Repository: accumulo-testing Updated Branches: refs/heads/master ac5b271ca -> 4f8220640
ACCUMULO-4510 Created 'accumulo-testing' command * Runs randomwalk locally or builds shaded jar that is submitted to YARN * Added documentation for random walk Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/efaa7377 Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/efaa7377 Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/efaa7377 Branch: refs/heads/master Commit: efaa7377c88a4716a7abf8f30e025ee7d68d6499 Parents: ac5b271 Author: Mike Walch <[email protected]> Authored: Wed Dec 28 14:09:53 2016 -0500 Committer: Mike Walch <[email protected]> Committed: Mon Jan 9 16:21:03 2017 -0500 ---------------------------------------------------------------------- .travis.yml | 22 +++ README.md | 53 ++++++ bin/accumulo-testing | 101 +++++++++++ conf/.gitignore | 2 + conf/accumulo-testing-env.sh.example | 25 +++ conf/accumulo-testing.properties.example | 44 +++++ conf/log4j.properties.example | 27 +++ core/pom.xml | 4 - .../apache/accumulo/testing/core/TestProps.java | 53 ++++++ .../testing/core/randomwalk/Environment.java | 49 +----- .../testing/core/randomwalk/Framework.java | 22 +-- .../testing/core/randomwalk/Module.java | 16 +- .../core/randomwalk/multitable/CopyTable.java | 3 +- .../randomwalk/sequential/MapRedVerify.java | 3 +- .../main/resources/randomwalk/modules/Bulk.xml | 2 +- .../resources/randomwalk/modules/Concurrent.xml | 2 +- .../randomwalk/modules/Conditional.xml | 3 +- .../main/resources/randomwalk/modules/Image.xml | 2 +- .../resources/randomwalk/modules/MultiTable.xml | 2 +- .../resources/randomwalk/modules/Security.xml | 2 +- .../resources/randomwalk/modules/Sequential.xml | 2 +- .../main/resources/randomwalk/modules/Shard.xml | 2 +- .../resources/randomwalk/modules/unit/Basic.xml | 2 +- .../randomwalk/modules/unit/Simple.xml | 2 +- pom.xml | 47 ++++++ yarn/.gitignore | 2 + yarn/pom.xml | 110 ++++++++++++ .../testing/yarn/YarnAccumuloTestRunner.java | 169 +++++++++++++++++++ yarn/src/main/resources/logback.xml | 31 ++++ 29 files changed, 724 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3688f4a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +language: java +cache: + directories: + - $HOME/.m2 +jdk: + - oraclejdk8 +install: true +script: mvn clean verify http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 9b70451..49b38a9 100644 --- a/README.md +++ b/README.md @@ -1 +1,54 @@ # Apache Accumulo Testing Suite + +The Apache Accumulo testing suite contains applications that test and verify the +correctness of Accumulo. + +## Installation + +In order to run the Apache Accumulo testing suite, you will need Java 8 and Maven installed +on your machine as well as an Accumulo instance to use for testing. + +1. First clone this repository. + + git clone [email protected]:apache/accumulo-testing.git + cd accumulo-testing + +2. All configuation files for the test suite are in `conf/`. Only the `accumulo-testing.properties` + configuration file needs to be created and edited as all other configuration files are optional. + In `accumulo-testing.properites`, review the properties with `test.common.*` prefix as these are + used by all tests. + + cd conf/ + cp accumulo-testing.properties.example accumulo-testing.properties + vim accumulo-testing.properties + +3. Tests are run using the `accumulo-testing` command which is located in the `bin/` + directory. Run this command without any arguments to view its usage and see available tests. + + ./bin/accumulo-testing + +## Random walk test + +The random walk test generates client behavior on an Apache Accumulo instance by randomly walking a +graph of client operations. + +Before running random walk, review the `test.common.*` and `test.randomwalk.*` properties in +`accumulo-testing.properties` file. A test module must also be specified. See the [modules][modules] +directory for a list of available ones. + +The command below will start a single random walker in a local process using the [Image.xml][image] +module. + + ./bin/accumulo-testing rw-local Image.xml + +If you would like to run multiple, distributed random walkers, run the command below to start random +walkers in 5 containers in YARN using the Image.xml module. + + ./bin/accumulo-testing rw-yarn 5 Image.xml + +This command will create an application in YARN and exit when test is completed. While its +running, you can view logs for each random walker using the YARN resource manager. The YARN +application can be killed at any time using `ctrl-c` or via the resource manager. + +[modules]: core/src/main/resources/randomwalk/modules +[image]: core/src/main/resources/randomwalk/modules/Image.xml http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/bin/accumulo-testing ---------------------------------------------------------------------- diff --git a/bin/accumulo-testing b/bin/accumulo-testing new file mode 100755 index 0000000..e5c1063 --- /dev/null +++ b/bin/accumulo-testing @@ -0,0 +1,101 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +at_home=$( cd "$( dirname "$bin_dir" )" && pwd ) +at_version=2.0.0-SNAPSHOT + +function print_usage() { + cat <<EOF + +Usage: accumulo-testing <command> (<argument>) + +Possible commands: + rw-local <module> Runs randomwalk <module> in local java process + rw-yarn <num> <module> Runs randomwalk <module> in <num> containers on YARN +EOF +} + +if [ -f "$at_home/conf/accumulo-testing-env.sh" ]; then + . "$at_home"/conf/accumulo-testing-env.sh +else + . "$at_home"/conf/accumulo-testing-env.sh.example +fi + +if [ -z "$ACCUMULO_VERSION" ]; then + echo "ERROR: ACCUMULO_VERSION must be set conf/accumulo-testing-env.sh" + exit 1 +fi +if [ -z "$HADOOP_VERSION" ]; then + echo "ERROR: HADOOP_VERSION must be set conf/accumulo-testing-env.sh" + exit 1 +fi + +at_props="$at_home/conf/accumulo-testing.properties" +if [ ! -f "$at_props" ]; then + echo "Please create and edit accumulo-testing.properties in $at_home/conf" + exit 1 +fi + +log4j_config="$at_home/conf/log4j.properties" +if [ ! -f "$log4j_config" ]; then + log4j_config="$at_home/conf/log4j.properties.example" + if [ ! -f "$log4j_config" ]; then + echo "Could not find logj4.properties or log4j.properties.example in $at_home/conf" + exit 1 + fi +fi + +function build_shade_jar() { + export at_shaded_jar="$at_home/core/target/accumulo-testing-core-$at_version-shaded.jar" + if [ ! -f "$at_shaded_jar" ]; then + echo "Building $at_shaded_jar" + cd "$at_home" || exit 1 + mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION" + fi +} + +randomwalk_main="org.apache.accumulo.testing.core.randomwalk.Framework" +case "$1" in +rw-local) + if [ -z "$2" ]; then + echo "ERROR: <module> needs to be set" + print_usage + exit 1 + fi + build_shade_jar + java -Dlog4j.configuration="file:$log4j_config" -cp "$at_shaded_jar" "$randomwalk_main" "$at_props" "$2" + ;; +rw-yarn) + if [ -z "$2" ]; then + echo "ERROR: <num> needs to be set" + print_usage + exit 1 + fi + if [ -z "$3" ]; then + echo "ERROR: <module> needs to be set" + print_usage + exit 1 + fi + build_shade_jar + mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRandomWalkTest -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3" + ;; +*) + echo "Unknown command: $1" + print_usage + exit 1 +esac http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/.gitignore ---------------------------------------------------------------------- diff --git a/conf/.gitignore b/conf/.gitignore new file mode 100644 index 0000000..336f891 --- /dev/null +++ b/conf/.gitignore @@ -0,0 +1,2 @@ +/accumulo-testing.properties +/accumulo-testing-env.sh http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/accumulo-testing-env.sh.example ---------------------------------------------------------------------- diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example new file mode 100644 index 0000000..24f8571 --- /dev/null +++ b/conf/accumulo-testing-env.sh.example @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hadoop +# ====== +test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop +test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop + +# Accumulo +# ======== +# Set the Accumulo version that should be included in the shaded jar +export ACCUMULO_VERSION=`accumulo version` +export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'` http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/accumulo-testing.properties.example ---------------------------------------------------------------------- diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example new file mode 100644 index 0000000..05cbaf5 --- /dev/null +++ b/conf/accumulo-testing.properties.example @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################### +# Common properties +################### + +# Accumulo instance name +test.common.accumulo.instance=instance +# Accumulo username +test.common.accumulo.username=root +# Accumulo password +test.common.accumulo.password=secret +# Accumulo keytab +#test.common.accumulo.keytab= +# Zookeeper connection string +test.common.zookeepers=localhost:2181 +# Memory (in MB) given to each container (if running in YARN) +test.common.yarn.container.memory.mb=1024 +# Number of cores given to each container (if running in YARN) +test.common.yarn.container.cores=1 + +######################## +# Random walk properties +######################## + +# Max memory for multi-table batch writer +test.randomwalk.bw.max.mem=100000000 +# Max latency in milliseconds for multi-table batch writer +test.randomwalk.bw.max.latency=600000 +# Number of write thread for multi-table batch writer +test.randomwalk.bw.num.threads=4 http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/log4j.properties.example ---------------------------------------------------------------------- diff --git a/conf/log4j.properties.example b/conf/log4j.properties.example new file mode 100644 index 0000000..4cbfd34 --- /dev/null +++ b/conf/log4j.properties.example @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=INFO, CA +log4j.appender.CA=org.apache.log4j.ConsoleAppender +log4j.appender.CA.layout=org.apache.log4j.PatternLayout +log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c] %-5p: %m%n + +log4j.logger.org.apache.accumulo=WARN +log4j.logger.org.apache.accumulo.testing=DEBUG +log4j.logger.org.apache.curator=ERROR +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.hadoop.mapreduce=ERROR +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.zookeeper=ERROR http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 83998d5..742ff89 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -31,10 +31,6 @@ <dependencies> <dependency> - <groupId>com.beust</groupId> - <artifactId>jcommander</artifactId> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java new file mode 100644 index 0000000..f8ce9ca --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.testing.core; + +public class TestProps { + + private static final String PREFIX = "test."; + private static final String RANDOMWALK = PREFIX + "randomwalk."; + private static final String COMMON = PREFIX + "common."; + + /** Common properties **/ + // Zookeeper connection string + public static final String ZOOKEEPERS = COMMON + "zookeepers"; + // Accumulo instance name + public static final String ACCUMULO_INSTANCE = COMMON + "accumulo.instance"; + // Accumulo username + public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username"; + // Accumulo password + public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password"; + // Accumulo keytab + public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab"; + // Memory (in MB) given to each YARN container + public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb"; + // Number of cores given to each YARN container + public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores"; + + + /** Random walk properties **/ + // Number of random walker (if running in YARN) + public static final String RW_NUM_WALKERS = RANDOMWALK + "num.walkers"; + // Max memory for multi-table batch writer + public static final String RW_BW_MAX_MEM = RANDOMWALK + "bw.max.mem"; + // Max latency in milliseconds for multi-table batch writer + public static final String RW_BW_MAX_LATENCY = RANDOMWALK + "bw.max.latency"; + // Number of write thread for multi-table batch writer + public static final String RW_BW_NUM_THREADS = RANDOMWALK + "bw.num.threads"; + +} http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java index 5684353..92a5de0 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java @@ -35,6 +35,7 @@ import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.testing.core.TestProps; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,38 +45,6 @@ import org.slf4j.LoggerFactory; * for creating client-side objects. This class is not thread-safe. */ public class Environment { - /** - * The configuration property key for a username. - */ - public static final String KEY_USERNAME = "USERNAME"; - /** - * The configuration property key for a password. - */ - public static final String KEY_PASSWORD = "PASSWORD"; - /** - * The configuration property key for a keytab - */ - public static final String KEY_KEYTAB = "KEYTAB"; - /** - * The configuration property key for the instance name. - */ - public static final String KEY_INSTANCE = "INSTANCE"; - /** - * The configuration property key for the comma-separated list of ZooKeepers. - */ - public static final String KEY_ZOOKEEPERS = "ZOOKEEPERS"; - /** - * The configuration property key for the maximum memory for the multi-table batch writer. - */ - public static final String KEY_MAX_MEM = "MAX_MEM"; - /** - * The configuration property key for the maximum latency, in milliseconds, for the multi-table batch writer. - */ - public static final String KEY_MAX_LATENCY = "MAX_LATENCY"; - /** - * The configuration property key for the number of write threads for the multi-table batch writer. - */ - public static final String KEY_NUM_THREADS = "NUM_THREADS"; private static final Logger log = LoggerFactory.getLogger(Environment.class); @@ -123,7 +92,7 @@ public class Environment { * @return username */ public String getUserName() { - return p.getProperty(KEY_USERNAME); + return p.getProperty(TestProps.ACCUMULO_USERNAME); } /** @@ -132,7 +101,7 @@ public class Environment { * @return password */ public String getPassword() { - return p.getProperty(KEY_PASSWORD); + return p.getProperty(TestProps.ACCUMULO_PASSWORD); } /** @@ -141,7 +110,7 @@ public class Environment { * @return path to keytab */ public String getKeytab() { - return p.getProperty(KEY_KEYTAB); + return p.getProperty(TestProps.ACCUMULO_KEYTAB); } /** @@ -186,8 +155,8 @@ public class Environment { */ public Instance getInstance() { if (instance == null) { - String instance = p.getProperty(KEY_INSTANCE); - String zookeepers = p.getProperty(KEY_ZOOKEEPERS); + String instance = p.getProperty(TestProps.ACCUMULO_INSTANCE); + String zookeepers = p.getProperty(TestProps.ZOOKEEPERS); this.instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers)); } return instance; @@ -216,9 +185,9 @@ public class Environment { */ public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException { if (mtbw == null) { - long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM)); - long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY)); - int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS)); + long maxMem = Long.parseLong(p.getProperty(TestProps.RW_BW_MAX_MEM)); + long maxLatency = Long.parseLong(p.getProperty(TestProps.RW_BW_MAX_LATENCY)); + int numThreads = Integer.parseInt(p.getProperty(TestProps.RW_BW_NUM_THREADS)); mtbw = getConnector().createMultiTableBatchWriter( new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads)); } http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java index 1a5700e..a9f1d68 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java @@ -16,15 +16,12 @@ */ package org.apache.accumulo.testing.core.randomwalk; -import java.io.File; import java.io.FileInputStream; import java.util.HashMap; import java.util.Properties; import org.apache.log4j.Logger; -import com.beust.jcommander.Parameter; - public class Framework { private static final Logger log = Logger.getLogger(Framework.class); @@ -63,7 +60,7 @@ public class Framework { * Name of node * @return Node specified by id */ - public Node getNode(String id) throws Exception { + Node getNode(String id) throws Exception { // check for node in nodes if (nodes.containsKey(id)) { @@ -71,9 +68,9 @@ public class Framework { } // otherwise create and put in nodes - Node node = null; + Node node; if (id.endsWith(".xml")) { - node = new Module(new File("/randomwalk/modules/" + id)); + node = new Module(id); } else { node = (Test) Class.forName(id).newInstance(); } @@ -81,13 +78,6 @@ public class Framework { return node; } - static class Opts extends org.apache.accumulo.core.cli.Help { - @Parameter(names = "--configDir", required = true, description = "directory containing the test configuration") - String configDir; - @Parameter(names = "--module", required = true, description = "the name of the module to run") - String module; - } - public static void main(String[] args) throws Exception { if (args.length != 2) { @@ -100,10 +90,12 @@ public class Framework { props.load(fis); fis.close(); + log.info("Running random walk test with module: " + args[1]); + State state = new State(); Environment env = new Environment(props); - int retval = getInstance().run(args[1], state, env); + getInstance().run(args[1], state, env); - System.exit(retval); + log.info("Test finished"); } } http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java index 1a3d059..3206906 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java @@ -18,7 +18,6 @@ package org.apache.accumulo.testing.core.randomwalk; import static java.nio.charset.StandardCharsets.UTF_8; -import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -170,12 +169,12 @@ public class Module extends Node { private HashMap<String,String> prefixes = new HashMap<>(); private HashMap<String,AdjList> adjMap = new HashMap<>(); private HashMap<String,Set<String>> aliasMap = new HashMap<>(); - private final File xmlFile; + private final String id; private String initNodeId; private Fixture fixture = null; - public Module(File xmlFile) throws Exception { - this.xmlFile = xmlFile; + public Module(String id) throws Exception { + this.id = id; loadFromXml(); } @@ -414,7 +413,7 @@ public class Module extends Node { @Override public String toString() { - return xmlFile.toString(); + return id; } private String getFullName(String name) { @@ -497,12 +496,13 @@ public class Module extends Node { dbf.setSchema(moduleSchema); // parse the document + try { docbuilder = dbf.newDocumentBuilder(); - d = docbuilder.parse(xmlFile); + d = docbuilder.parse(this.getClass().getResourceAsStream("/randomwalk/modules/" + id)); } catch (Exception e) { - log.error("Failed to parse: " + xmlFile, e); - throw new Exception("Failed to parse: " + xmlFile); + log.error("Failed to parse xml at randomwalk/modules/" + id, e); + throw new Exception("Failed to parse xml at randomwalk/modules/" + id); } // parse packages http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java index 6552161..b67f3d7 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java @@ -23,6 +23,7 @@ import java.util.TreeSet; import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.testing.core.TestProps; import org.apache.accumulo.testing.core.randomwalk.Environment; import org.apache.accumulo.testing.core.randomwalk.State; import org.apache.accumulo.testing.core.randomwalk.Test; @@ -64,7 +65,7 @@ public class CopyTable extends Test { } args[4] = srcTableName; args[5] = env.getInstance().getInstanceName(); - args[6] = env.getConfigProperty("ZOOKEEPERS"); + args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS); args[7] = dstTableName; log.debug("copying " + srcTableName + " to " + dstTableName); http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java index 58d44d4..5113973 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.testing.core.TestProps; import org.apache.accumulo.testing.core.randomwalk.Environment; import org.apache.accumulo.testing.core.randomwalk.State; import org.apache.accumulo.testing.core.randomwalk.Test; @@ -46,7 +47,7 @@ public class MapRedVerify extends Test { } args[4] = state.getString("seqTableName"); args[5] = env.getInstance().getInstanceName(); - args[6] = env.getConfigProperty("ZOOKEEPERS"); + args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS); args[7] = args[4] + "_MR"; if (ToolRunner.run(CachedConfiguration.getInstance(), new MapRedVerifyTool(), args) != 0) { http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Bulk.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Bulk.xml b/core/src/main/resources/randomwalk/modules/Bulk.xml index 35e2a67..9ffa76a 100644 --- a/core/src/main/resources/randomwalk/modules/Bulk.xml +++ b/core/src/main/resources/randomwalk/modules/Bulk.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="bulk" value="org.apache.accumulo.test.randomwalk.bulk"/> +<package prefix="bulk" value="org.apache.accumulo.testing.core.randomwalk.bulk"/> <init id="bulk.Setup"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Concurrent.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml index 36ea53c..ea96b27 100644 --- a/core/src/main/resources/randomwalk/modules/Concurrent.xml +++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="ct" value="org.apache.accumulo.test.randomwalk.concurrent"/> +<package prefix="ct" value="org.apache.accumulo.testing.core.randomwalk.concurrent"/> <fixture id="ct.ConcurrentFixture"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Conditional.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Conditional.xml b/core/src/main/resources/randomwalk/modules/Conditional.xml index 54ff7ab..3c380dc 100644 --- a/core/src/main/resources/randomwalk/modules/Conditional.xml +++ b/core/src/main/resources/randomwalk/modules/Conditional.xml @@ -17,8 +17,7 @@ --> <module> -<package prefix="ct" value="org.apache.accumulo.test.randomwalk.conditional"/> - +<package prefix="ct" value="org.apache.accumulo.testing.core.randomwalk.conditional"/> <init id="ct.Setup"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Image.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Image.xml b/core/src/main/resources/randomwalk/modules/Image.xml index 7561895..0b41d14 100644 --- a/core/src/main/resources/randomwalk/modules/Image.xml +++ b/core/src/main/resources/randomwalk/modules/Image.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="image" value="org.apache.accumulo.test.randomwalk.image"/> +<package prefix="image" value="org.apache.accumulo.testing.core.randomwalk.image"/> <fixture id="image.ImageFixture"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/MultiTable.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/MultiTable.xml b/core/src/main/resources/randomwalk/modules/MultiTable.xml index 55f6590..fa42c3f 100644 --- a/core/src/main/resources/randomwalk/modules/MultiTable.xml +++ b/core/src/main/resources/randomwalk/modules/MultiTable.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="mt" value="org.apache.accumulo.test.randomwalk.multitable"/> +<package prefix="mt" value="org.apache.accumulo.testing.core.randomwalk.multitable"/> <fixture id="mt.MultiTableFixture"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Security.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Security.xml b/core/src/main/resources/randomwalk/modules/Security.xml index 9e9ef9f..422b63e 100644 --- a/core/src/main/resources/randomwalk/modules/Security.xml +++ b/core/src/main/resources/randomwalk/modules/Security.xml @@ -12,7 +12,7 @@ <module> <package prefix="security" - value="org.apache.accumulo.test.randomwalk.security" /> + value="org.apache.accumulo.testing.core.randomwalk.security" /> <fixture id="security.SecurityFixture" /> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Sequential.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Sequential.xml b/core/src/main/resources/randomwalk/modules/Sequential.xml index 454e75f..ce4abd0 100644 --- a/core/src/main/resources/randomwalk/modules/Sequential.xml +++ b/core/src/main/resources/randomwalk/modules/Sequential.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="seq" value="org.apache.accumulo.test.randomwalk.sequential"/> +<package prefix="seq" value="org.apache.accumulo.testing.core.randomwalk.sequential"/> <fixture id="seq.SequentialFixture"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Shard.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/Shard.xml b/core/src/main/resources/randomwalk/modules/Shard.xml index eb23f37..e8e8654 100644 --- a/core/src/main/resources/randomwalk/modules/Shard.xml +++ b/core/src/main/resources/randomwalk/modules/Shard.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="shard" value="org.apache.accumulo.test.randomwalk.shard"/> +<package prefix="shard" value="org.apache.accumulo.testing.core.randomwalk.shard"/> <fixture id="shard.ShardFixture"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/unit/Basic.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/unit/Basic.xml b/core/src/main/resources/randomwalk/modules/unit/Basic.xml index 2dead02..f81f132 100644 --- a/core/src/main/resources/randomwalk/modules/unit/Basic.xml +++ b/core/src/main/resources/randomwalk/modules/unit/Basic.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/> +<package prefix="test" value="org.apache.accumulo.testing.core.randomwalk.unit"/> <init id="test.CreateTable"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/unit/Simple.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/randomwalk/modules/unit/Simple.xml b/core/src/main/resources/randomwalk/modules/unit/Simple.xml index cad940e..6eb685a 100644 --- a/core/src/main/resources/randomwalk/modules/unit/Simple.xml +++ b/core/src/main/resources/randomwalk/modules/unit/Simple.xml @@ -17,7 +17,7 @@ --> <module> -<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/> +<package prefix="test" value="org.apache.accumulo.testing.core.randomwalk.unit"/> <init id="dummy.all"/> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a2efdf4..2c72faf 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ <modules> <module>core</module> + <module>yarn</module> </modules> <properties> @@ -41,6 +42,7 @@ <hadoop.version>2.6.4</hadoop.version> <zookeeper.version>3.4.6</zookeeper.version> <slf4j.version>1.7.21</slf4j.version> + <twill.version>0.9.0</twill.version> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> </properties> @@ -58,6 +60,11 @@ <version>1.48</version> </dependency> <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-testing-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>commons-configuration</groupId> <artifactId>commons-configuration</artifactId> <version>1.6</version> @@ -93,6 +100,46 @@ <version>${hadoop.version}</version> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-api</artifactId> + <version>${twill.version}</version> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-ext</artifactId> + <version>${twill.version}</version> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-yarn</artifactId> + <version>${twill.version}</version> + </dependency> + <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.version}</version> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/.gitignore ---------------------------------------------------------------------- diff --git a/yarn/.gitignore b/yarn/.gitignore new file mode 100644 index 0000000..17bb010 --- /dev/null +++ b/yarn/.gitignore @@ -0,0 +1,2 @@ +/target/ +/*.iml http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/pom.xml ---------------------------------------------------------------------- diff --git a/yarn/pom.xml b/yarn/pom.xml new file mode 100644 index 0000000..650c3b9 --- /dev/null +++ b/yarn/pom.xml @@ -0,0 +1,110 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-testing</artifactId> + <version>2.0.0-SNAPSHOT</version> + </parent> + + <artifactId>accumulo-testing-yarn</artifactId> + <packaging>jar</packaging> + + <name>Apache Accumulo Testing YARN</name> + + <dependencies> + <dependency> + <groupId>com.beust</groupId> + <artifactId>jcommander</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <!-- Twill requires version 13.0.1 --> + <version>13.0.1</version> + </dependency> + <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-testing-core</artifactId> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-ext</artifactId> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.twill</groupId> + <artifactId>twill-yarn</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>yarn-test-runner</id> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>java</goal> + </goals> + <phase>compile</phase> + <configuration> + <mainClass>org.apache.accumulo.testing.yarn.YarnAccumuloTestRunner</mainClass> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + +</project> http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java ---------------------------------------------------------------------- diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java new file mode 100644 index 0000000..e50dbb4 --- /dev/null +++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.testing.yarn; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.common.base.Preconditions; +import org.apache.accumulo.testing.core.TestProps; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.twill.api.ResourceSpecification; +import org.apache.twill.api.TwillApplication; +import org.apache.twill.api.TwillController; +import org.apache.twill.api.TwillRunnerService; +import org.apache.twill.api.TwillSpecification; +import org.apache.twill.ext.BundledJarRunnable; +import org.apache.twill.ext.BundledJarRunner; +import org.apache.twill.yarn.YarnTwillRunnerService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class YarnAccumuloTestRunner { + + private static final Logger LOG = LoggerFactory.getLogger(YarnAccumuloTestRunner.class); + + private static class YarnTestApp implements TwillApplication { + + private TestRunnerOpts opts; + private Properties props; + + YarnTestApp(TestRunnerOpts opts, Properties props) { + this.opts = opts; + this.props = props; + } + + @Override + public TwillSpecification configure() { + + int numCores = Integer.valueOf(props.getProperty(TestProps.YARN_CONTAINER_CORES)); + int memory = Integer.valueOf(props.getProperty(TestProps.YARN_CONTAINER_MEMORY_MB)); + + ResourceSpecification resourceSpec = ResourceSpecification.Builder.with() + .setVirtualCores(numCores).setMemory(memory, ResourceSpecification.SizeUnit.MEGA) + .setInstances(opts.numContainers).build(); + + File jarFile = new File(opts.jarPath); + File testProps = new File(opts.testProps); + File log4jProps = new File(opts.logProps); + + return TwillSpecification.Builder.with() + .setName(opts.testName) + .withRunnable() + .add("BundledJarRunnable", new BundledJarRunnable(), resourceSpec) + .withLocalFiles() + .add(jarFile.getName(), jarFile.toURI(), false) + .add(testProps.getName(), testProps.toURI()) + .add(log4jProps.getName(), log4jProps.toURI()) + .apply() + .anyOrder() + .build(); + } + } + + private static class TestRunnerOpts { + + @Parameter(names={"--testName", "-t"}, required = true, description = "Test name") + String testName; + + @Parameter(names={"--numContainers", "-n"}, required = true, description = "Test name") + int numContainers; + + @Parameter(names={"--jar", "-j"}, required = true, description = "Bundled jar path") + String jarPath; + + @Parameter(names={"--main", "-m"}, required = true, description = "Main class") + String mainClass; + + @Parameter(names={"--testProps", "-p"}, required = true, description = "Test properties path") + String testProps; + + @Parameter(names={"--logProps", "-l"}, required = true, description = "Log properties path") + String logProps; + + @Parameter(names={"--args", "-a"}, variableArity = true, description = "Main class args") + List<String> mainArgs = new ArrayList<>(); + } + + private static void verifyPath(String path) { + File f = new File(path); + Preconditions.checkState(f.exists()); + Preconditions.checkState(f.canRead()); + } + + public static void main(String[] args) throws Exception { + + TestRunnerOpts opts = new TestRunnerOpts(); + new JCommander(opts, args); + + verifyPath(opts.jarPath); + verifyPath(opts.testProps); + verifyPath(opts.logProps); + + String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new); + BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(opts.jarPath, "/lib", + opts.mainClass, mainArgs); + + Properties props = new Properties(); + FileInputStream fis = new FileInputStream(opts.testProps); + props.load(fis); + fis.close(); + String zookeepers = props.getProperty(TestProps.ZOOKEEPERS); + + final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), + zookeepers); + twillRunner.start(); + + final TwillController controller = twillRunner.prepare( + new YarnTestApp(opts, props)) + .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName()) + .withArguments("BundledJarRunnable", arguments.toArray()) + .start(); + + final AtomicBoolean done = new AtomicBoolean(false); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + if (!done.get()) { + controller.kill(); + } + } finally { + twillRunner.stop(); + } + })); + + LOG.info("Waiting for {} to finish in YARN...", opts.testName); + LOG.info("Press ctrl-c to kill {} in YARN", opts.testName); + + try { + controller.awaitTerminated(); + done.set(true); + } catch (ExecutionException e) { + LOG.error("Exception during execution", e); + throw e; + } + LOG.info("{} finished", opts.testName); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/src/main/resources/logback.xml ---------------------------------------------------------------------- diff --git a/yarn/src/main/resources/logback.xml b/yarn/src/main/resources/logback.xml new file mode 100644 index 0000000..7a8e3b8 --- /dev/null +++ b/yarn/src/main/resources/logback.xml @@ -0,0 +1,31 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. +--> +<configuration> + + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <!-- encoders are assigned the type + ch.qos.logback.classic.encoder.PatternLayoutEncoder by default --> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> + </encoder> + </appender> + + <logger name="org.apache.twill" level="warn"/> + <logger name="org.apache.twill.yarn.YarnTwillRunnerService" level="info"/> + + <root level="info"> + <appender-ref ref="STDOUT" /> + </root> +</configuration>
