This is an automated email from the ASF dual-hosted git repository. mwalch pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push: new e4ec7e3 Updates due to Accumulo 2.0 changes (#35) e4ec7e3 is described below commit e4ec7e3c9b790dd3c07ebe48e5b90d376d315232 Author: Mike Walch <mwa...@apache.org> AuthorDate: Tue Sep 25 15:20:23 2018 -0400 Updates due to Accumulo 2.0 changes (#35) - using accumulo-client.properties to create clients --- bin/accumulo-testing | 14 +- conf/accumulo-testing-env.sh.example | 16 ++- conf/accumulo-testing.properties.example | 20 --- .../org/apache/accumulo/testing/core/TestEnv.java | 149 ++++++--------------- .../apache/accumulo/testing/core/TestProps.java | 28 +--- .../core/continuous/ContinuousBatchWalker.java | 17 ++- .../testing/core/continuous/ContinuousEnv.java | 17 ++- .../testing/core/continuous/ContinuousIngest.java | 25 ++-- .../testing/core/continuous/ContinuousMoru.java | 23 ++-- .../testing/core/continuous/ContinuousScanner.java | 14 +- .../testing/core/continuous/ContinuousVerify.java | 27 ++-- .../testing/core/continuous/ContinuousWalk.java | 11 +- .../testing/core/continuous/CreateTable.java | 11 +- .../testing/core/continuous/UndefinedAnalyzer.java | 1 - .../accumulo/testing/core/ingest/TestIngest.java | 2 - .../accumulo/testing/core/ingest/VerifyIngest.java | 1 - .../testing/core/merkle/cli/ComputeRootHash.java | 1 - .../testing/core/merkle/cli/GenerateHashes.java | 1 - .../testing/core/merkle/cli/ManualComparison.java | 1 - .../testing/core/merkle/ingest/RandomWorkload.java | 1 - .../testing/core/randomwalk/Framework.java | 13 +- .../accumulo/testing/core/randomwalk/Module.java | 2 +- .../testing/core/randomwalk/RandWalkEnv.java | 14 +- .../core/randomwalk/concurrent/Replication.java | 2 - .../core/randomwalk/image/ImageFixture.java | 2 - .../core/randomwalk/multitable/CopyTable.java | 15 +-- .../core/randomwalk/multitable/CopyTool.java | 71 +--------- .../core/randomwalk/security/AlterTable.java | 16 +-- .../core/randomwalk/security/AlterTablePerm.java | 24 ++-- .../core/randomwalk/security/Authenticate.java | 8 +- .../core/randomwalk/security/ChangePass.java | 12 +- .../core/randomwalk/security/CreateTable.java | 10 +- .../core/randomwalk/security/CreateUser.java | 8 +- .../core/randomwalk/security/DropTable.java | 12 +- .../testing/core/randomwalk/security/DropUser.java | 8 +- .../core/randomwalk/security/SecurityFixture.java | 6 +- .../testing/core/randomwalk/security/SetAuths.java | 8 +- .../testing/core/randomwalk/security/TableOp.java | 23 ++-- .../testing/core/randomwalk/security/Validate.java | 1 - .../core/randomwalk/sequential/MapRedVerify.java | 15 +-- .../randomwalk/sequential/MapRedVerifyTool.java | 61 +-------- .../randomwalk/sequential/SequentialFixture.java | 1 - .../testing/core/randomwalk/shard/BulkInsert.java | 1 - .../apache/accumulo/testing/core/stress/Scan.java | 1 - .../apache/accumulo/testing/core/stress/Write.java | 1 - .../core/randomwalk/ReplicationRandomWalkIT.java | 27 +--- pom.xml | 8 +- .../testing/yarn/YarnAccumuloTestRunner.java | 47 +++---- 48 files changed, 264 insertions(+), 533 deletions(-) diff --git a/bin/accumulo-testing b/bin/accumulo-testing index 34fadb0..465e11b 100755 --- a/bin/accumulo-testing +++ b/bin/accumulo-testing @@ -128,7 +128,7 @@ function start_agitator() { datanode_log="${log_base}_datanode-agitator" master_cmd="nohup ${libexec}/master-agitator.pl $AGTR_MASTER_KILL_SLEEP_TIME $AGTR_MASTER_RESTART_SLEEP_TIME" tserver_cmd="nohup ${libexec}/tserver-agitator.pl $AGTR_TSERVER_KILL_SLEEP_TIME $AGTR_TSERVER_RESTART_SLEEP_TIME $AGTR_TSERVER_MIN_KILL $AGTR_TSERVER_MAX_KILL" - datanode_cmd="nohup ${libexec}/datanode-agitator.pl $AGTR_DATANODE_KILL_SLEEP_TIME $AGTR_DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $AGTR_DATANODE_MIN_KILL $AGTR_DATANODE_MAX_KILL" + datanode_cmd="nohup ${libexec}/datanode-agitator.pl $AGTR_DATANODE_KILL_SLEEP_TIME $AGTR_DATANODE_RESTART_SLEEP_TIME $HADOOP_HOME $AGTR_DATANODE_MIN_KILL $AGTR_DATANODE_MAX_KILL" [[ -n $AGITATOR_USER ]] || AGITATOR_USER=$(whoami) if [[ $AGITATOR_USER == root ]]; then @@ -187,7 +187,7 @@ randomwalk_main="org.apache.accumulo.testing.core.randomwalk.Framework" case "$1" in ci-createtable) build_shade_jar - java -Dlog4j.configuration="file:$log4j_config" org.apache.accumulo.testing.core.continuous.CreateTable "$at_props" + java -Dlog4j.configuration="file:$log4j_config" org.apache.accumulo.testing.core.continuous.CreateTable "$at_props" "$ACCUMULO_CLIENT_PROPS" ;; ci-local) if [ -z "$2" ]; then @@ -197,7 +197,7 @@ ci-local) fi determine_app_main "$2" build_shade_jar - java -Dlog4j.configuration="file:$log4j_config" "$ci_main" "$at_props" + java -Dlog4j.configuration="file:$log4j_config" "$ci_main" "$at_props" "$ACCUMULO_CLIENT_PROPS" ;; ci-yarn) if [ -z "$2" ]; then @@ -212,7 +212,7 @@ ci-yarn) fi determine_app_main "$3" build_shade_jar - mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloCITest-$3 -j $at_shaded_jar -m $ci_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties" + mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloCITest-$3 -j $at_shaded_jar -m $ci_main -n $2 -p $at_props -c $ACCUMULO_CLIENT_PROPS -l $log4j_config -a ./accumulo-testing.properties ./accumulo-client.properties" ;; ci-mapred) if [ -z "$2" ]; then @@ -222,7 +222,7 @@ ci-mapred) fi determine_mapred_main "$2" build_shade_jar - "$HADOOP_PREFIX"/bin/yarn jar "$at_shaded_jar" "$ci_main" "$at_props" + "$HADOOP_HOME"/bin/yarn jar "$at_shaded_jar" "$ci_main" "$at_props" "$ACCUMULO_CLIENT_PROPS" ;; rw-local) if [ -z "$2" ]; then @@ -231,7 +231,7 @@ rw-local) exit 1 fi build_shade_jar - java -Dlog4j.configuration="file:$log4j_config" "$randomwalk_main" "$at_props" "$2" + java -Dlog4j.configuration="file:$log4j_config" "$randomwalk_main" "$at_props" "$ACCUMULO_CLIENT_PROPS" "$2" ;; rw-yarn) if [ -z "$2" ]; then @@ -245,7 +245,7 @@ rw-yarn) exit 1 fi build_shade_jar - mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRWTest-$3 -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3" + mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRWTest-$3 -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -c $ACCUMULO_CLIENT_PROPS -l $log4j_config -a ./accumulo-testing.properties ./accumulo-client.properties $3" ;; agitator) case "$2" in diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example index 677c23d..3876718 100644 --- a/conf/accumulo-testing-env.sh.example +++ b/conf/accumulo-testing-env.sh.example @@ -15,18 +15,22 @@ # General # ======= -test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop + +## Hadoop installation +export HADOOP_HOME="${HADOOP_HOME:-/path/to/hadoop}" +## Accumulo installation +export ACCUMULO_HOME="${ACCUMULO_HOME:-/path/to/accumulo}" +## Path to Accumulo client properties +export ACCUMULO_CLIENT_PROPS="$ACCUMULO_HOME/conf/accumulo-client.properties" # Versions set below will be what is included in the shaded jar -export ACCUMULO_VERSION=`accumulo version` -export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'` +export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`" +export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`" export ZOOKEEPER_VERSION=3.4.9 # Make sure Hadoop configuration directory is on the classpath -export CLASSPATH=$HADOOP_PREFIX/etc/hadoop +export CLASSPATH=$HADOOP_HOME/etc/hadoop # Agitator # ======== -# Agitator needs know where Accumulo is installed -test -z "$ACCUMULO_HOME" && export ACCUMULO_HOME=/path/to/accumulo # Accumulo user AGTR_ACCUMULO_USER=$(whoami) # Time (in minutes) between killing Accumulo masters diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example index 4fff104..b084c4c 100644 --- a/conf/accumulo-testing.properties.example +++ b/conf/accumulo-testing.properties.example @@ -17,24 +17,6 @@ # Common properties ################### -# Accumulo instance name -test.common.accumulo.instance=instance -# Accumulo username -test.common.accumulo.username=root -# Accumulo password -test.common.accumulo.password=secret -# Max memory (in bytes) each batch writer will use to buffer writes -test.common.accumulo.bw.max.memory.bytes=100000000 -# Max latency (in milliseconds) that each batch writer will buffer data -test.common.accumulo.bw.max.latency.ms=600000 -# Number of write thread for each batch writer -test.common.accumulo.bw.num.threads=4 -# Number of threads used by batch scanner -test.common.accumulo.bs.num.threads=8 -# Number of key/value entries to pull during scan -test.common.accumulo.scanner.batch.size=1000 -# Accumulo keytab -#test.common.accumulo.keytab= # HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml test.common.hdfs.root=hdfs://localhost:8020 # YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in Hadoop's yarn-site.xml @@ -43,8 +25,6 @@ test.common.yarn.resource.manager=localhost test.common.yarn.container.memory.mb=1024 # Number of cores given to each container (if running in YARN) test.common.yarn.container.cores=1 -# Zookeeper connection string -test.common.zookeepers=localhost:2181 ################################### # Continuous ingest test properties diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java index 48266f5..5ec94c7 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/TestEnv.java @@ -2,62 +2,55 @@ package org.apache.accumulo.testing.core; import static java.util.Objects.requireNonNull; -import java.io.File; -import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.Properties; -import java.util.concurrent.TimeUnit; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.ClientInfo; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; -import org.apache.accumulo.core.client.security.tokens.KerberosToken; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.ClientProperty; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; public class TestEnv { - protected final Properties p; - private Instance instance = null; - private Connector connector = null; + protected final Properties testProps; + private String clientPropsPath; + private ClientInfo info; + private AccumuloClient client = null; private Configuration hadoopConfig = null; - /** - * Creates new test environment using provided properties - * - * @param p - * Properties - */ - public TestEnv(Properties p) { - requireNonNull(p); - this.p = p; + public TestEnv(String testPropsPath, String clientPropsPath) { + requireNonNull(testPropsPath); + requireNonNull(clientPropsPath); + this.testProps = TestProps.loadFromFile(testPropsPath); + this.clientPropsPath = clientPropsPath; + this.info = ClientInfo.from(TestProps.loadFromFile(clientPropsPath)); } /** - * Gets a copy of the configuration properties. - * - * @return a copy of the configuration properties + * @return a copy of the test properties */ - public Properties copyConfigProperties() { - return new Properties(p); + public Properties getTestProperties() { + return new Properties(testProps); } /** - * Gets a configuration property. - * - * @param key - * key - * @return property value + * @return a test property value given a key */ - public String getConfigProperty(String key) { - return p.getProperty(key); + public String getTestProperty(String key) { + return testProps.getProperty(key); + } + + public String getClientPropsPath() { + return clientPropsPath; + } + + public ClientInfo getInfo() { + return info; } /** @@ -66,7 +59,7 @@ public class TestEnv { * @return username */ public String getAccumuloUserName() { - return p.getProperty(TestProps.ACCUMULO_USERNAME); + return info.getPrincipal(); } /** @@ -75,16 +68,11 @@ public class TestEnv { * @return password */ public String getAccumuloPassword() { - return p.getProperty(TestProps.ACCUMULO_PASSWORD); - } - - /** - * Gets the configured keytab. - * - * @return path to keytab - */ - public String getAccumuloKeytab() { - return p.getProperty(TestProps.ACCUMULO_KEYTAB); + String authType = info.getProperties().getProperty(ClientProperty.AUTH_TYPE.getKey()); + if (authType.equals("password")) { + return info.getProperties().getProperty(ClientProperty.AUTH_TOKEN.getKey()); + } + return null; } /** @@ -114,79 +102,28 @@ public class TestEnv { * Gets an authentication token based on the configured password. */ public AuthenticationToken getToken() { - String password = getAccumuloPassword(); - if (null != password) { - return new PasswordToken(getAccumuloPassword()); - } - String keytab = getAccumuloKeytab(); - if (null != keytab) { - File keytabFile = new File(keytab); - if (!keytabFile.exists() || !keytabFile.isFile()) { - throw new IllegalArgumentException("Provided keytab is not a normal file: " + keytab); - } - try { - UserGroupInformation.loginUserFromKeytab(getAccumuloUserName(), keytabFile.getAbsolutePath()); - return new KerberosToken(); - } catch (IOException e) { - throw new RuntimeException("Failed to login", e); - } - } - throw new IllegalArgumentException("Must provide password or keytab in configuration"); - } - - public String getAccumuloInstanceName() { - return p.getProperty(TestProps.ACCUMULO_INSTANCE); + return info.getAuthenticationToken(); } public String getHdfsRoot() { - return p.getProperty(TestProps.HDFS_ROOT); + return testProps.getProperty(TestProps.HDFS_ROOT); } public String getYarnResourceManager() { - return p.getProperty(TestProps.YARN_RESOURCE_MANAGER); - } - - public String getZookeepers() { - return p.getProperty(TestProps.ZOOKEEPERS); - } - - public ClientConfiguration getClientConfiguration() { - return ClientConfiguration.loadDefault().withInstance(getAccumuloInstanceName()).withZkHosts(getZookeepers()); - } - - /** - * Gets an Accumulo instance object. The same instance is reused after the first call. - */ - public Instance getAccumuloInstance() { - if (instance == null) { - this.instance = new ZooKeeperInstance(getClientConfiguration()); - } - return instance; + return testProps.getProperty(TestProps.YARN_RESOURCE_MANAGER); } /** * Gets an Accumulo connector. The same connector is reused after the first call. */ - public Connector getAccumuloConnector() throws AccumuloException, AccumuloSecurityException { - if (connector == null) { - connector = getAccumuloInstance().getConnector(getAccumuloUserName(), getToken()); + public AccumuloClient getAccumuloClient() throws AccumuloException, AccumuloSecurityException { + if (client == null) { + client = Accumulo.newClient().usingClientInfo(info).build(); } - return connector; + return client; } - public BatchWriterConfig getBatchWriterConfig() { - int numThreads = Integer.parseInt(p.getProperty(TestProps.ACCUMULO_BW_NUM_THREADS)); - long maxLatency = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_LATENCY_MS)); - long maxMemory = Long.parseLong(p.getProperty(TestProps.ACCUMULO_BW_MAX_MEM_BYTES)); - - BatchWriterConfig config = new BatchWriterConfig(); - config.setMaxWriteThreads(numThreads); - config.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS); - config.setMaxMemory(maxMemory); - return config; - } - - public int getScannerBatchSize() { - return Integer.parseInt(p.getProperty(TestProps.ACCUMULO_SCANNER_BATCH_SIZE)); + public Connector getAccumuloConnector() throws AccumuloException, AccumuloSecurityException { + return Connector.from(getAccumuloClient()); } } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java index 638d2db..3b2c98f 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java @@ -34,26 +34,6 @@ public class TestProps { private static final String CI_VERIFY = CI + "verify."; /** Common properties **/ - // Zookeeper connection string - public static final String ZOOKEEPERS = COMMON + "zookeepers"; - // Accumulo instance name - public static final String ACCUMULO_INSTANCE = COMMON + "accumulo.instance"; - // Accumulo username - public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username"; - // Accumulo password - public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password"; - // Max memory (in bytes) each batch writer will use to buffer writes - public static final String ACCUMULO_BW_MAX_MEM_BYTES = COMMON + "accumulo.bw.max.memory.bytes"; - // Max the maximum time (in ms) each batch writer will buffer data - public static final String ACCUMULO_BW_MAX_LATENCY_MS = COMMON + "accumulo.bw.max.latency.ms"; - // Number of threads each batch writer will use to write data - public static final String ACCUMULO_BW_NUM_THREADS = COMMON + "accumulo.bw.num.threads"; - // Number of thread for each batch scanner - public static final String ACCUMULO_BS_NUM_THREADS = COMMON + "accumulo.bw.num.threads"; - // Number of key/value entries to pull during scan - public static final String ACCUMULO_SCANNER_BATCH_SIZE = COMMON + "accumulo.scanner.batch.size"; - // Accumulo keytab - public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab"; // HDFS root path. Should match 'fs.defaultFS' property in Hadoop's core-site.xml public static final String HDFS_ROOT = COMMON + "hdfs.root"; // YARN resource manager hostname. Should match 'yarn.resourcemanager.hostname' property in @@ -132,8 +112,12 @@ public class TestProps { // Location in HDFS to store output public static final String CI_VERIFY_OUTPUT_DIR = CI_VERIFY + "output.dir"; - public static Properties loadFromFile(String propsFilePath) throws IOException { - return loadFromStream(new FileInputStream(propsFilePath)); + public static Properties loadFromFile(String propsFilePath) { + try { + return loadFromStream(new FileInputStream(propsFilePath)); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } public static Properties loadFromStream(FileInputStream fis) throws IOException { diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java index 88134a0..2917800 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousBatchWalker.java @@ -21,7 +21,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; -import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -42,22 +41,22 @@ public class ContinuousBatchWalker { public static void main(String[] args) throws Exception { - Properties props = TestProps.loadFromFile(args[0]); - - ContinuousEnv env = new ContinuousEnv(props); + if (args.length != 2) { + System.err.println("Usage: ContinuousBatchWalker <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); Authorizations auths = env.getRandomAuthorizations(); Connector conn = env.getAccumuloConnector(); Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), auths); - int scanBatchSize = Integer.parseInt(props.getProperty(TestProps.CI_BW_BATCH_SIZE)); + int scanBatchSize = Integer.parseInt(env.getTestProperty(TestProps.CI_BW_BATCH_SIZE)); scanner.setBatchSize(scanBatchSize); Random r = new Random(); - int scanThreads = Integer.parseInt(props.getProperty(TestProps.ACCUMULO_BS_NUM_THREADS)); - while (true) { - BatchScanner bs = conn.createBatchScanner(env.getAccumuloTableName(), auths, scanThreads); + BatchScanner bs = conn.createBatchScanner(env.getAccumuloTableName(), auths); Set<Text> batch = getBatch(scanner, env.getRowMin(), env.getRowMax(), scanBatchSize, r); List<Range> ranges = new ArrayList<>(batch.size()); @@ -68,7 +67,7 @@ public class ContinuousBatchWalker { runBatchScan(scanBatchSize, bs, batch, ranges); - int bwSleepMs = Integer.parseInt(props.getProperty(TestProps.CI_BW_SLEEP_MS)); + int bwSleepMs = Integer.parseInt(env.getTestProperty(TestProps.CI_BW_SLEEP_MS)); sleepUninterruptibly(bwSleepMs, TimeUnit.MILLISECONDS); } } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java index 7907ffd..76d6705 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousEnv.java @@ -3,7 +3,6 @@ package org.apache.accumulo.testing.core.continuous; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Properties; import java.util.Random; import org.apache.accumulo.core.security.Authorizations; @@ -14,8 +13,8 @@ class ContinuousEnv extends TestEnv { private List<Authorizations> authList; - ContinuousEnv(Properties props) { - super(props); + ContinuousEnv(String testPropsPath, String clientPropsPath) { + super(testPropsPath, clientPropsPath); } /** @@ -23,7 +22,7 @@ class ContinuousEnv extends TestEnv { */ private List<Authorizations> getAuthList() { if (authList == null) { - String authValue = p.getProperty(TestProps.CI_COMMON_AUTHS); + String authValue = testProps.getProperty(TestProps.CI_COMMON_AUTHS); if (authValue == null || authValue.trim().isEmpty()) { authList = Collections.singletonList(Authorizations.EMPTY); } else { @@ -45,22 +44,22 @@ class ContinuousEnv extends TestEnv { } long getRowMin() { - return Long.parseLong(p.getProperty(TestProps.CI_INGEST_ROW_MIN)); + return Long.parseLong(testProps.getProperty(TestProps.CI_INGEST_ROW_MIN)); } long getRowMax() { - return Long.parseLong(p.getProperty(TestProps.CI_INGEST_ROW_MAX)); + return Long.parseLong(testProps.getProperty(TestProps.CI_INGEST_ROW_MAX)); } int getMaxColF() { - return Integer.parseInt(p.getProperty(TestProps.CI_INGEST_MAX_CF)); + return Integer.parseInt(testProps.getProperty(TestProps.CI_INGEST_MAX_CF)); } int getMaxColQ() { - return Integer.parseInt(p.getProperty(TestProps.CI_INGEST_MAX_CQ)); + return Integer.parseInt(testProps.getProperty(TestProps.CI_INGEST_MAX_CQ)); } String getAccumuloTableName() { - return p.getProperty(TestProps.CI_COMMON_ACCUMULO_TABLE); + return testProps.getProperty(TestProps.CI_COMMON_ACCUMULO_TABLE); } } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java index 4afd00c..729fc53 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java @@ -99,14 +99,14 @@ public class ContinuousIngest { public static void main(String[] args) throws Exception { - if (args.length != 1) { - System.err.println("Usage: ContinuousIngest <propsPath>"); + if (args.length != 2) { + System.err.println("Usage: ContinuousIngest <testPropsPath> <clientPropsPath>"); System.exit(-1); } - Properties props = TestProps.loadFromFile(args[0]); + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); - String vis = props.getProperty(TestProps.CI_INGEST_VISIBILITIES); + String vis = env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES); if (vis == null) { visibilities = Collections.singletonList(new ColumnVisibility()); } else { @@ -116,8 +116,6 @@ public class ContinuousIngest { } } - ContinuousEnv env = new ContinuousEnv(props); - long rowMin = env.getRowMin(); long rowMax = env.getRowMax(); if (rowMin < 0 || rowMax < 0 || rowMax <= rowMin) { @@ -130,7 +128,7 @@ public class ContinuousIngest { throw new TableNotFoundException(null, tableName, "Consult the README and create the table before starting ingest."); } - BatchWriter bw = conn.createBatchWriter(tableName, env.getBatchWriterConfig()); + BatchWriter bw = conn.createBatchWriter(tableName); bw = Trace.wrapAll(bw, new CountSampler(1024)); Random r = new Random(); @@ -159,12 +157,13 @@ public class ContinuousIngest { int maxColF = env.getMaxColF(); int maxColQ = env.getMaxColQ(); - boolean checksum = Boolean.parseBoolean(props.getProperty(TestProps.CI_INGEST_CHECKSUM)); - long numEntries = Long.parseLong(props.getProperty(TestProps.CI_INGEST_CLIENT_ENTRIES)); + boolean checksum = Boolean.parseBoolean(env.getTestProperty(TestProps.CI_INGEST_CHECKSUM)); + long numEntries = Long.parseLong(env.getTestProperty(TestProps.CI_INGEST_CLIENT_ENTRIES)); - if (pauseEnabled(props)) { + Properties testProps = env.getTestProperties(); + if (pauseEnabled(testProps)) { lastPauseNs = System.nanoTime(); - pauseWaitSec = getPauseWaitSec(props, r); + pauseWaitSec = getPauseWaitSec(testProps, r); log.info("PAUSING enabled"); log.info("INGESTING for " + pauseWaitSec + "s"); } @@ -208,7 +207,7 @@ public class ContinuousIngest { lastFlushTime = flush(bw, count, flushInterval, lastFlushTime); if (count >= numEntries) break out; - pauseCheck(props, r); + pauseCheck(testProps, r); } // create one big linked list, this makes all of the first inserts @@ -222,7 +221,7 @@ public class ContinuousIngest { lastFlushTime = flush(bw, count, flushInterval, lastFlushTime); if (count >= numEntries) break out; - pauseCheck(props, r); + pauseCheck(testProps, r); } bw.close(); } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java index 81f9bd3..abaccc9 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousMoru.java @@ -19,7 +19,6 @@ package org.apache.accumulo.testing.core.continuous; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; -import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -113,19 +112,21 @@ public class ContinuousMoru extends Configured implements Tool { @Override public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException { - Properties props = TestProps.loadFromFile(args[0]); - ContinuousEnv env = new ContinuousEnv(props); + if (args.length != 2) { + System.err.println("Usage: ContinuousMoru <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); - AccumuloInputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken()); + AccumuloInputFormat.setClientInfo(job, env.getInfo()); AccumuloInputFormat.setInputTableName(job, env.getAccumuloTableName()); - AccumuloInputFormat.setZooKeeperInstance(job, env.getClientConfiguration()); - int maxMaps = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_MAX_MAPS)); + int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS)); // set up ranges try { @@ -139,11 +140,9 @@ public class ContinuousMoru extends Configured implements Tool { job.setMapperClass(CMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.setBatchWriterOptions(job, env.getBatchWriterConfig()); - AccumuloOutputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken()); + AccumuloOutputFormat.setClientInfo(job, env.getInfo()); AccumuloOutputFormat.setCreateTables(job, true); AccumuloOutputFormat.setDefaultTableName(job, env.getAccumuloTableName()); - AccumuloOutputFormat.setZooKeeperInstance(job, env.getClientConfiguration()); Configuration conf = job.getConfiguration(); conf.setLong(MIN, env.getRowMin()); @@ -157,7 +156,11 @@ public class ContinuousMoru extends Configured implements Tool { } public static void main(String[] args) throws Exception { - ContinuousEnv env = new ContinuousEnv(TestProps.loadFromFile(args[0])); + if (args.length != 2) { + System.err.println("Usage: ContinuousMoru <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousMoru(), args); if (res != 0) System.exit(res); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java index 1531269..c051691 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousScanner.java @@ -20,7 +20,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Iterator; import java.util.Map.Entry; -import java.util.Properties; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -38,9 +37,11 @@ import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterrup public class ContinuousScanner { public static void main(String[] args) throws Exception { - - Properties props = TestProps.loadFromFile(args[0]); - ContinuousEnv env = new ContinuousEnv(props); + if (args.length != 2) { + System.err.println("Usage: ContinuousScanner <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); Random r = new Random(); @@ -49,10 +50,9 @@ public class ContinuousScanner { Connector conn = env.getAccumuloConnector(); Authorizations auths = env.getRandomAuthorizations(); Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), auths); - scanner.setBatchSize(env.getScannerBatchSize()); - int numToScan = Integer.parseInt(props.getProperty(TestProps.CI_SCANNER_ENTRIES)); - int scannerSleepMs = Integer.parseInt(props.getProperty(TestProps.CI_SCANNER_SLEEP_MS)); + int numToScan = Integer.parseInt(env.getTestProperty(TestProps.CI_SCANNER_ENTRIES)); + int scannerSleepMs = Integer.parseInt(env.getTestProperty(TestProps.CI_SCANNER_SLEEP_MS)); double delta = Math.min(.05, .05 / (numToScan / 1000.0)); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java index d343c9b..6eef2fe 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousVerify.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.Properties; import java.util.Random; import java.util.Set; @@ -140,20 +139,22 @@ public class ContinuousVerify extends Configured implements Tool { @Override public int run(String[] args) throws Exception { - - Properties props = TestProps.loadFromFile(args[0]); - ContinuousEnv env = new ContinuousEnv(props); + if (args.length != 2) { + System.err.println("Usage: ContinuousVerify <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(AccumuloInputFormat.class); - boolean scanOffline = Boolean.parseBoolean(props.getProperty(TestProps.CI_VERIFY_SCAN_OFFLINE)); + boolean scanOffline = Boolean.parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE)); String tableName = env.getAccumuloTableName(); - int maxMaps = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_MAX_MAPS)); - int reducers = Integer.parseInt(props.getProperty(TestProps.CI_VERIFY_REDUCERS)); - String outputDir = props.getProperty(TestProps.CI_VERIFY_OUTPUT_DIR); + int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS)); + int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS)); + String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR); Set<Range> ranges; String clone = ""; @@ -174,8 +175,7 @@ public class ContinuousVerify extends Configured implements Tool { AccumuloInputFormat.setRanges(job, ranges); AccumuloInputFormat.setAutoAdjustRanges(job, false); - AccumuloInputFormat.setConnectorInfo(job, env.getAccumuloUserName(), env.getToken()); - AccumuloInputFormat.setZooKeeperInstance(job, env.getClientConfiguration()); + AccumuloInputFormat.setClientInfo(job, env.getInfo()); job.setMapperClass(CMapper.class); job.setMapOutputKeyClass(LongWritable.class); @@ -199,8 +199,11 @@ public class ContinuousVerify extends Configured implements Tool { } public static void main(String[] args) throws Exception { - - ContinuousEnv env = new ContinuousEnv(TestProps.loadFromFile(args[0])); + if (args.length != 2) { + System.err.println("Usage: ContinuousVerify <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); int res = ToolRunner.run(env.getHadoopConfiguration(), new ContinuousVerify(), args); if (res != 0) diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java index 49c10c9..aa5b899 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousWalk.java @@ -20,7 +20,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.util.ArrayList; import java.util.Map.Entry; -import java.util.Properties; import java.util.Random; import java.util.zip.CRC32; @@ -46,9 +45,11 @@ public class ContinuousWalk { } public static void main(String[] args) throws Exception { - - Properties props = TestProps.loadFromFile(args[0]); - ContinuousEnv env = new ContinuousEnv(props); + if (args.length != 2) { + System.err.println("Usage: ContinuousWalk <testPropsPath> <clientPropsPath>"); + System.exit(-1); + } + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); Connector conn = env.getAccumuloConnector(); @@ -56,7 +57,7 @@ public class ContinuousWalk { ArrayList<Value> values = new ArrayList<>(); - int sleepTime = Integer.parseInt(props.getProperty(TestProps.CI_WALKER_SLEEP_MS)); + int sleepTime = Integer.parseInt(env.getTestProperty(TestProps.CI_WALKER_SLEEP_MS)); while (true) { Scanner scanner = ContinuousUtil.createScanner(conn, env.getAccumuloTableName(), env.getRandomAuthorizations()); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java index bac8368..c2386cb 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/CreateTable.java @@ -16,7 +16,6 @@ */ package org.apache.accumulo.testing.core.continuous; -import java.util.Properties; import java.util.SortedSet; import java.util.TreeSet; @@ -28,13 +27,11 @@ public class CreateTable { public static void main(String[] args) throws Exception { - if (args.length != 1) { - System.err.println("Usage: CreateTable <propsPath>"); + if (args.length != 2) { + System.err.println("Usage: CreateTable <testPropsPath> <clientPropsPath>"); System.exit(-1); } - - Properties props = TestProps.loadFromFile(args[0]); - ContinuousEnv env = new ContinuousEnv(props); + ContinuousEnv env = new ContinuousEnv(args[0], args[1]); Connector conn = env.getAccumuloConnector(); String tableName = env.getAccumuloTableName(); @@ -43,7 +40,7 @@ public class CreateTable { System.exit(-1); } - int numTablets = Integer.parseInt(props.getProperty(TestProps.CI_COMMON_ACCUMULO_NUM_TABLETS)); + int numTablets = Integer.parseInt(env.getTestProperty(TestProps.CI_COMMON_ACCUMULO_NUM_TABLETS)); if (numTablets < 1) { System.err.println("ERROR: numTablets < 1"); System.exit(-1); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java index 70df0c0..ccd5a78 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java @@ -38,7 +38,6 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.cli.BatchScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java index 2d43bee..57a3cf1 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/TestIngest.java @@ -30,13 +30,11 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.impl.TabletServerBatchWriter; import org.apache.accumulo.core.client.security.SecurityErrorCode; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ConstraintViolationSummary; import org.apache.accumulo.core.data.Key; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java index be3de45..2fd62b1 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/VerifyIngest.java @@ -25,7 +25,6 @@ import org.apache.accumulo.core.cli.ScannerOpts; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java index a2286bb..b3d664e 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ComputeRootHash.java @@ -26,7 +26,6 @@ import org.apache.accumulo.core.cli.ClientOnRequiredTable; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java index 4c9ac70..126ea5f 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/GenerateHashes.java @@ -39,7 +39,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java index 5531d81..2ff6046 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java @@ -21,7 +21,6 @@ import java.util.Map.Entry; import org.apache.accumulo.core.cli.ClientOpts; import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java index 2a0f26d..1c63575 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java @@ -23,7 +23,6 @@ import org.apache.accumulo.core.cli.ClientOnDefaultTable; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.hadoop.io.Text; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java index 43c66b4..96083fa 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java @@ -19,7 +19,6 @@ package org.apache.accumulo.testing.core.randomwalk; import java.util.HashMap; import java.util.Properties; -import org.apache.accumulo.testing.core.TestProps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,18 +80,16 @@ public class Framework { public static void main(String[] args) throws Exception { - if (args.length != 2) { - System.out.println("Usage: Framework <propsPath> <module>"); + if (args.length != 3) { + System.out.println("Usage: Framework <testPropsPath> <clientPropsPath> <module>"); System.exit(-1); } - Properties props = TestProps.loadFromFile(args[0]); - - log.info("Running random walk test with module: " + args[1]); + log.info("Running random walk test with module: " + args[2]); State state = new State(); - RandWalkEnv env = new RandWalkEnv(props); - getInstance().run(args[1], state, env); + RandWalkEnv env = new RandWalkEnv(args[0], args[1]); + getInstance().run(args[2], state, env); log.info("Test finished"); } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java index 68ffe6a..ef26591 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java @@ -342,7 +342,7 @@ public class Module extends Node { log.debug(" " + entry.getKey() + ": " + entry.getValue()); } log.debug("Overall Configuration Properties"); - for (Entry<Object,Object> entry : env.copyConfigProperties().entrySet()) { + for (Entry<Object,Object> entry : env.getTestProperties().entrySet()) { log.debug(" " + entry.getKey() + ": " + entry.getValue()); } log.debug("State information"); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java index f85f560..207b364 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/RandWalkEnv.java @@ -16,8 +16,6 @@ */ package org.apache.accumulo.testing.core.randomwalk; -import java.util.Properties; - import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.MultiTableBatchWriter; @@ -35,14 +33,8 @@ public class RandWalkEnv extends TestEnv { private MultiTableBatchWriter mtbw = null; - /** - * Creates a new test environment. - * - * @param p - * configuration properties - */ - public RandWalkEnv(Properties p) { - super(p); + public RandWalkEnv(String testPropsPath, String clientPropsPath) { + super(testPropsPath, clientPropsPath); } /** @@ -56,7 +48,7 @@ public class RandWalkEnv extends TestEnv { */ public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException { if (mtbw == null) { - mtbw = getAccumuloConnector().createMultiTableBatchWriter(getBatchWriterConfig()); + mtbw = getAccumuloClient().createMultiTableBatchWriter(); } return mtbw; } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java index 9324861..4d987d4 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java @@ -40,8 +40,6 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.TableOperations; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java index cd9fd1e..c70efaa 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java @@ -26,7 +26,6 @@ import java.util.SortedSet; import java.util.TreeSet; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableExistsException; @@ -44,7 +43,6 @@ public class ImageFixture extends Fixture { public void setUp(State state, RandWalkEnv env) throws Exception { Connector conn = env.getAccumuloConnector(); - Instance instance = env.getAccumuloInstance(); SortedSet<Text> splits = new TreeSet<>(); for (int i = 1; i < 256; i++) { diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java index 6417913..cbaf0b8 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java @@ -21,7 +21,6 @@ import java.util.Properties; import java.util.Random; import java.util.TreeSet; -import org.apache.accumulo.testing.core.TestProps; import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv; import org.apache.accumulo.testing.core.randomwalk.State; import org.apache.accumulo.testing.core.randomwalk.Test; @@ -53,16 +52,10 @@ public class CopyTable extends Test { int nextId = ((Integer) state.get("nextId")).intValue(); String dstTableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId); - String[] args = new String[6]; - args[0] = env.getAccumuloUserName(); - args[1] = env.getAccumuloPassword(); - if (null == args[1]) { - args[1] = env.getAccumuloKeytab(); - } - args[2] = srcTableName; - args[3] = env.getAccumuloInstance().getInstanceName(); - args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS); - args[5] = dstTableName; + String[] args = new String[3]; + args[0] = env.getClientPropsPath(); + args[1] = srcTableName; + args[2] = dstTableName; log.debug("copying " + srcTableName + " to " + dstTableName); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java index 5c979d3..df265eb 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java @@ -18,26 +18,18 @@ package org.apache.accumulo.testing.core.randomwalk.multitable; import java.io.IOException; -import org.apache.accumulo.core.client.ClientConfiguration; -import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.client.admin.DelegationTokenConfig; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.ClientInfo; import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; -import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; -import org.apache.accumulo.core.client.security.tokens.KerberosToken; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.SystemPermission; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,70 +56,21 @@ public class CopyTool extends Configured implements Tool { return 1; } - ClientConfiguration clientConf = ClientConfiguration.create().withInstance(args[3]).withZkHosts(args[4]); - + ClientInfo info = Accumulo.newClient().usingProperties(args[0]).info(); job.setInputFormatClass(AccumuloInputFormat.class); - AccumuloInputFormat.setInputTableName(job, args[2]); + AccumuloInputFormat.setClientInfo(job, info); + AccumuloInputFormat.setInputTableName(job, args[1]); AccumuloInputFormat.setScanAuthorizations(job, Authorizations.EMPTY); - AccumuloInputFormat.setZooKeeperInstance(job, clientConf); - - final String principal; - final AuthenticationToken token; - if (Boolean.parseBoolean(clientConf.get(ClientProperty.INSTANCE_RPC_SASL_ENABLED))) { - // Use the Kerberos creds to request a DelegationToken for MapReduce - // to use - // We could use the specified keytab (args[1]), but we're already - // logged in and don't need to, so we can just use the current user - KerberosToken kt = new KerberosToken(); - try { - UserGroupInformation user = UserGroupInformation.getCurrentUser(); - if (!user.hasKerberosCredentials()) { - throw new IllegalStateException("Expected current user to have Kerberos credentials"); - } - - // Get the principal via UGI - principal = user.getUserName(); - - // Connector w/ the Kerberos creds - ZooKeeperInstance inst = new ZooKeeperInstance(clientConf); - Connector conn = inst.getConnector(principal, kt); - - // Do the explicit check to see if the user has the permission - // to get a delegation token - if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) { - log.error(principal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name() - + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's" - + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials."); - throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token"); - } - - // Fetch a delegation token from Accumulo - token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig()); - - } catch (Exception e) { - final String msg = "Failed to acquire DelegationToken for use with MapReduce"; - log.error(msg, e); - throw new RuntimeException(msg, e); - } - } else { - // Simple principal + password - principal = args[0]; - token = new PasswordToken(args[1]); - } - - AccumuloInputFormat.setConnectorInfo(job, principal, token); - AccumuloOutputFormat.setConnectorInfo(job, principal, token); job.setMapperClass(SeqMapClass.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); - job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); + AccumuloOutputFormat.setClientInfo(job, info); AccumuloOutputFormat.setCreateTables(job, true); - AccumuloOutputFormat.setDefaultTableName(job, args[5]); - AccumuloOutputFormat.setZooKeeperInstance(job, clientConf); + AccumuloOutputFormat.setDefaultTableName(job, args[2]); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java index 1283fd7..c973e11 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java @@ -19,9 +19,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.net.InetAddress; import java.util.Properties; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.security.SecurityErrorCode; @@ -36,15 +36,15 @@ public class AlterTable extends Test { @Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { String systemUser = WalkingSecurity.get(state, env).getSysUserName(); - Connector conn = env.getAccumuloInstance().getConnector(systemUser, WalkingSecurity.get(state, env).getSysToken()); + AccumuloClient client = env.getAccumuloClient().changeUser(systemUser, WalkingSecurity.get(state, env).getSysToken()); String tableName = WalkingSecurity.get(state, env).getTableName(); boolean exists = WalkingSecurity.get(state, env).getTableExists(); boolean hasPermission; try { - hasPermission = conn.securityOperations().hasTablePermission(systemUser, tableName, TablePermission.ALTER_TABLE) - || conn.securityOperations().hasSystemPermission(systemUser, SystemPermission.ALTER_TABLE); + hasPermission = client.securityOperations().hasTablePermission(systemUser, tableName, TablePermission.ALTER_TABLE) + || client.securityOperations().hasSystemPermission(systemUser, SystemPermission.ALTER_TABLE); } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST)) { if (exists) @@ -58,13 +58,13 @@ public class AlterTable extends Test { String newTableName = String.format("security_%s_%s_%d", InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_"), env.getPid(), System.currentTimeMillis()); - renameTable(conn, state, env, tableName, newTableName, hasPermission, exists); + renameTable(client, state, env, tableName, newTableName, hasPermission, exists); } - public static void renameTable(Connector conn, State state, RandWalkEnv env, String oldName, String newName, boolean hasPermission, boolean tableExists) + public static void renameTable(AccumuloClient client, State state, RandWalkEnv env, String oldName, String newName, boolean hasPermission, boolean tableExists) throws AccumuloSecurityException, AccumuloException, TableExistsException { try { - conn.tableOperations().rename(oldName, newName); + client.tableOperations().rename(oldName, newName); } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (hasPermission) @@ -72,7 +72,7 @@ public class AlterTable extends Test { else return; } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { - if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami())) + if (WalkingSecurity.get(state, env).userPassTransient(client.whoami())) return; } throw new AccumuloException("Got unexpected ae error code", ae); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java index 94a972d..553c766 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java @@ -19,9 +19,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; import java.util.Random; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.admin.SecurityOperations; import org.apache.accumulo.core.client.security.SecurityErrorCode; @@ -76,8 +76,8 @@ public class AlterTablePerm extends Test { sourceUser = env.getAccumuloUserName(); sourceToken = env.getToken(); } - Connector conn = env.getAccumuloInstance().getConnector(sourceUser, sourceToken); - SecurityOperations secOps = conn.securityOperations(); + AccumuloClient client = env.getAccumuloClient().changeUser(sourceUser, sourceToken); + SecurityOperations secOps = client.securityOperations(); try { canGive = secOps.hasSystemPermission(sourceUser, SystemPermission.ALTER_TABLE) || secOps.hasTablePermission(sourceUser, tableName, TablePermission.GRANT); @@ -97,7 +97,7 @@ public class AlterTablePerm extends Test { try { boolean res; if (hasPerm != (res = env.getAccumuloConnector().securityOperations().hasTablePermission(target, tableName, tabPerm))) - throw new AccumuloException("Test framework and accumulo are out of sync for user " + conn.whoami() + " for perm " + tabPerm.name() + throw new AccumuloException("Test framework and accumulo are out of sync for user " + client.whoami() + " for perm " + tabPerm.name() + " with local vs. accumulo being " + hasPerm + " " + res); if (hasPerm) @@ -122,17 +122,17 @@ public class AlterTablePerm extends Test { } } - boolean trans = WalkingSecurity.get(state, env).userPassTransient(conn.whoami()); + boolean trans = WalkingSecurity.get(state, env).userPassTransient(client.whoami()); if ("take".equals(action)) { try { - conn.securityOperations().revokeTablePermission(target, tableName, tabPerm); + client.securityOperations().revokeTablePermission(target, tableName, tabPerm); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case GRANT_INVALID: throw new AccumuloException("Got a grant invalid on non-System.GRANT option", ae); case PERMISSION_DENIED: if (canGive) - throw new AccumuloException(conn.whoami() + " failed to revoke permission to " + target + " when it should have worked", ae); + throw new AccumuloException(client.whoami() + " failed to revoke permission to " + target + " when it should have worked", ae); return; case USER_DOESNT_EXIST: if (userExists) @@ -144,7 +144,7 @@ public class AlterTablePerm extends Test { return; case BAD_CREDENTIALS: if (!trans) - throw new AccumuloException("Bad credentials for user " + conn.whoami()); + throw new AccumuloException("Bad credentials for user " + client.whoami()); return; default: throw new AccumuloException("Got unexpected exception", ae); @@ -153,14 +153,14 @@ public class AlterTablePerm extends Test { WalkingSecurity.get(state, env).revokeTablePermission(target, tableName, tabPerm); } else if ("give".equals(action)) { try { - conn.securityOperations().grantTablePermission(target, tableName, tabPerm); + client.securityOperations().grantTablePermission(target, tableName, tabPerm); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case GRANT_INVALID: throw new AccumuloException("Got a grant invalid on non-System.GRANT option", ae); case PERMISSION_DENIED: if (canGive) - throw new AccumuloException(conn.whoami() + " failed to give permission to " + target + " when it should have worked", ae); + throw new AccumuloException(client.whoami() + " failed to give permission to " + target + " when it should have worked", ae); return; case USER_DOESNT_EXIST: if (userExists) @@ -172,7 +172,7 @@ public class AlterTablePerm extends Test { return; case BAD_CREDENTIALS: if (!trans) - throw new AccumuloException("Bad credentials for user " + conn.whoami()); + throw new AccumuloException("Bad credentials for user " + client.whoami()); return; default: throw new AccumuloException("Got unexpected exception", ae); @@ -186,7 +186,7 @@ public class AlterTablePerm extends Test { if (!tableExists) throw new AccumuloException("Table shouldn't have existed, but apparently does"); if (!canGive) - throw new AccumuloException(conn.whoami() + " shouldn't have been able to grant privilege"); + throw new AccumuloException(client.whoami() + " shouldn't have been able to grant privilege"); } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java index 63105f4..037abbc 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java @@ -19,9 +19,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Arrays; import java.util.Properties; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.security.SystemPermission; @@ -40,7 +40,7 @@ public class Authenticate extends Test { String targetProp = props.getProperty("target"); boolean success = Boolean.parseBoolean(props.getProperty("valid")); - Connector conn = env.getAccumuloInstance().getConnector(principal, token); + AccumuloClient client = env.getAccumuloClient().changeUser(principal, token); String target; @@ -52,7 +52,7 @@ public class Authenticate extends Test { boolean exists = WalkingSecurity.get(state, env).userExists(target); // Copy so if failed it doesn't mess with the password stored in state byte[] password = Arrays.copyOf(WalkingSecurity.get(state, env).getUserPassword(target), WalkingSecurity.get(state, env).getUserPassword(target).length); - boolean hasPermission = conn.securityOperations().hasSystemPermission(principal, SystemPermission.SYSTEM) || principal.equals(target); + boolean hasPermission = client.securityOperations().hasSystemPermission(principal, SystemPermission.SYSTEM) || principal.equals(target); if (!success) for (int i = 0; i < password.length; i++) @@ -61,7 +61,7 @@ public class Authenticate extends Test { boolean result; try { - result = conn.securityOperations().authenticateUser(target, new PasswordToken(password)); + result = client.securityOperations().authenticateUser(target, new PasswordToken(password)); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case PERMISSION_DENIED: diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java index 585a2c1..b409892 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java @@ -19,9 +19,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; import java.util.Random; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.security.SystemPermission; @@ -45,7 +45,7 @@ public class ChangePass extends Test { principal = WalkingSecurity.get(state, env).getTabUserName(); token = WalkingSecurity.get(state, env).getTabToken(); } - Connector conn = env.getAccumuloInstance().getConnector(principal, token); + AccumuloClient client = env.getAccumuloClient().changeUser(principal, token); boolean hasPerm; boolean targetExists; @@ -56,7 +56,7 @@ public class ChangePass extends Test { targetExists = WalkingSecurity.get(state, env).userExists(target); - hasPerm = conn.securityOperations().hasSystemPermission(principal, SystemPermission.ALTER_USER) || principal.equals(target); + hasPerm = client.securityOperations().hasSystemPermission(principal, SystemPermission.ALTER_USER) || principal.equals(target); Random r = new Random(); @@ -66,7 +66,7 @@ public class ChangePass extends Test { PasswordToken newPass = new PasswordToken(newPassw); try { - conn.securityOperations().changeLocalUserPassword(target, newPass); + client.securityOperations().changeLocalUserPassword(target, newPass); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case PERMISSION_DENIED: @@ -78,8 +78,8 @@ public class ChangePass extends Test { throw new AccumuloException("User " + target + " doesn't exist and they SHOULD.", ae); return; case BAD_CREDENTIALS: - if (!WalkingSecurity.get(state, env).userPassTransient(conn.whoami())) - throw new AccumuloException("Bad credentials for user " + conn.whoami()); + if (!WalkingSecurity.get(state, env).userPassTransient(client.whoami())) + throw new AccumuloException("Bad credentials for user " + client.whoami()); return; default: throw new AccumuloException("Got unexpected exception", ae); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java index de11d62..e4e8190 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java @@ -18,9 +18,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.security.SecurityErrorCode; import org.apache.accumulo.core.security.SystemPermission; @@ -33,15 +33,15 @@ public class CreateTable extends Test { @Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { - Connector conn = env.getAccumuloInstance().getConnector(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken()); + AccumuloClient client = env.getAccumuloClient().changeUser(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken()); String tableName = WalkingSecurity.get(state, env).getTableName(); boolean exists = WalkingSecurity.get(state, env).getTableExists(); - boolean hasPermission = conn.securityOperations().hasSystemPermission(WalkingSecurity.get(state, env).getSysUserName(), SystemPermission.CREATE_TABLE); + boolean hasPermission = client.securityOperations().hasSystemPermission(WalkingSecurity.get(state, env).getSysUserName(), SystemPermission.CREATE_TABLE); try { - conn.tableOperations().create(tableName); + client.tableOperations().create(tableName); } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (hasPermission) @@ -69,7 +69,7 @@ public class CreateTable extends Test { } WalkingSecurity.get(state, env).initTable(tableName); for (TablePermission tp : TablePermission.values()) - WalkingSecurity.get(state, env).grantTablePermission(conn.whoami(), tableName, tp); + WalkingSecurity.get(state, env).grantTablePermission(client.whoami(), tableName, tp); if (!hasPermission) throw new AccumuloException("Didn't get Security Exception when we should have"); } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java index ca9afbe..f23f87c 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java @@ -18,9 +18,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv; @@ -32,15 +32,15 @@ public class CreateUser extends Test { @Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { String sysPrincipal = WalkingSecurity.get(state, env).getSysUserName(); - Connector conn = env.getAccumuloInstance().getConnector(sysPrincipal, WalkingSecurity.get(state, env).getSysToken()); + AccumuloClient client = env.getAccumuloClient().changeUser(sysPrincipal, WalkingSecurity.get(state, env).getSysToken()); String tableUserName = WalkingSecurity.get(state, env).getTabUserName(); boolean exists = WalkingSecurity.get(state, env).userExists(tableUserName); - boolean hasPermission = conn.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.CREATE_USER); + boolean hasPermission = client.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.CREATE_USER); PasswordToken tabUserPass = new PasswordToken("Super Sekret Table User Password"); try { - conn.securityOperations().createLocalUser(tableUserName, tabUserPass); + client.securityOperations().createLocalUser(tableUserName, tabUserPass); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case PERMISSION_DENIED: diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java index 66fc0e2..ddfee12 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java @@ -18,9 +18,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.security.SecurityErrorCode; @@ -50,16 +50,16 @@ public class DropTable extends Test { principal = WalkingSecurity.get(state, env).getSysUserName(); token = WalkingSecurity.get(state, env).getSysToken(); } - Connector conn = env.getAccumuloInstance().getConnector(principal, token); + AccumuloClient client = env.getAccumuloClient().changeUser(principal, token); String tableName = WalkingSecurity.get(state, env).getTableName(); boolean exists = WalkingSecurity.get(state, env).getTableExists(); try { - hasPermission = conn.securityOperations().hasTablePermission(principal, tableName, TablePermission.DROP_TABLE) - || conn.securityOperations().hasSystemPermission(principal, SystemPermission.DROP_TABLE); - conn.tableOperations().delete(tableName); + hasPermission = client.securityOperations().hasTablePermission(principal, tableName, TablePermission.DROP_TABLE) + || client.securityOperations().hasSystemPermission(principal, SystemPermission.DROP_TABLE); + client.tableOperations().delete(tableName); } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST)) { if (exists) @@ -76,7 +76,7 @@ public class DropTable extends Test { return; } } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { - if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami())) + if (WalkingSecurity.get(state, env).userPassTransient(client.whoami())) return; } throw new AccumuloException("Got unexpected ae error code", ae); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java index 933c26d..d5cf910 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java @@ -18,9 +18,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv; import org.apache.accumulo.testing.core.randomwalk.State; @@ -31,15 +31,15 @@ public class DropUser extends Test { @Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { String sysPrincipal = WalkingSecurity.get(state, env).getSysUserName(); - Connector conn = env.getAccumuloInstance().getConnector(sysPrincipal, WalkingSecurity.get(state, env).getSysToken()); + AccumuloClient client = env.getAccumuloClient().changeUser(sysPrincipal, WalkingSecurity.get(state, env).getSysToken()); String tableUserName = WalkingSecurity.get(state, env).getTabUserName(); boolean exists = WalkingSecurity.get(state, env).userExists(tableUserName); - boolean hasPermission = conn.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.DROP_USER); + boolean hasPermission = client.securityOperations().hasSystemPermission(sysPrincipal, SystemPermission.DROP_USER); try { - conn.securityOperations().dropLocalUser(tableUserName); + client.securityOperations().dropLocalUser(tableUserName); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case PERMISSION_DENIED: diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java index 1bffba0..8782fdb 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java @@ -19,10 +19,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.net.InetAddress; import java.util.Set; -import org.apache.accumulo.core.client.ClientConfiguration; -import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; @@ -36,8 +35,7 @@ public class SecurityFixture extends Fixture { public void setUp(State state, RandWalkEnv env) throws Exception { String secTableName, systemUserName, tableUserName, secNamespaceName; // A best-effort sanity check to guard against not password-based auth - ClientConfiguration clientConf = ClientConfiguration.loadDefault(); - if (Boolean.parseBoolean(clientConf.get(ClientProperty.INSTANCE_RPC_SASL_ENABLED))) { + if (env.getInfo().getProperties().getProperty(ClientProperty.AUTH_TYPE.getKey()).equals("kerberos")) { throw new IllegalStateException("Security module currently cannot support Kerberos/SASL instances"); } diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java index feee2d8..c309f35 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java @@ -19,9 +19,9 @@ package org.apache.accumulo.testing.core.randomwalk.security; import java.util.Properties; import java.util.Random; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SystemPermission; @@ -48,10 +48,10 @@ public class SetAuths extends Test { authPrincipal = env.getAccumuloUserName(); authToken = env.getToken(); } - Connector conn = env.getAccumuloInstance().getConnector(authPrincipal, authToken); + AccumuloClient client = env.getAccumuloClient().changeUser(authPrincipal, authToken); boolean exists = WalkingSecurity.get(state, env).userExists(target); - boolean hasPermission = conn.securityOperations().hasSystemPermission(authPrincipal, SystemPermission.ALTER_USER); + boolean hasPermission = client.securityOperations().hasSystemPermission(authPrincipal, SystemPermission.ALTER_USER); Authorizations auths; if (authsString.equals("_random")) { @@ -74,7 +74,7 @@ public class SetAuths extends Test { } try { - conn.securityOperations().changeUserAuthorizations(target, auths); + client.securityOperations().changeUserAuthorizations(target, auths); } catch (AccumuloSecurityException ae) { switch (ae.getSecurityErrorCode()) { case PERMISSION_DENIED: diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java index 2fee84f..3029542 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java @@ -19,7 +19,6 @@ package org.apache.accumulo.testing.core.randomwalk.security; import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Iterator; -import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Random; @@ -27,11 +26,11 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.UUID; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; @@ -58,9 +57,9 @@ public class TableOp extends Test { @Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { String tablePrincipal = WalkingSecurity.get(state, env).getTabUserName(); - Connector conn = env.getAccumuloInstance().getConnector(tablePrincipal, WalkingSecurity.get(state, env).getTabToken()); - TableOperations tableOps = conn.tableOperations(); - SecurityOperations secOps = conn.securityOperations(); + AccumuloClient client = env.getAccumuloClient().changeUser(tablePrincipal, WalkingSecurity.get(state, env).getTabToken()); + TableOperations tableOps = client.tableOperations(); + SecurityOperations secOps = client.securityOperations(); String action = props.getProperty("action", "_random"); TablePermission tp; @@ -85,12 +84,12 @@ public class TableOp extends Test { return; } Authorizations auths = secOps.getUserAuthorizations(tablePrincipal); - boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp); - boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(conn.whoami()); + boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(client.whoami(), tp); + boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(client.whoami()); Scanner scan = null; try { - scan = conn.createScanner(tableName, secOps.getUserAuthorizations(conn.whoami())); + scan = client.createScanner(tableName, secOps.getUserAuthorizations(client.whoami())); int seen = 0; Iterator<Entry<Key,Value>> iter = scan.iterator(); while (iter.hasNext()) { @@ -101,7 +100,7 @@ public class TableOp extends Test { throw new AccumuloException("Got data I should not be capable of seeing: " + k + " table " + tableName); } if (!canRead && !ambiguousZone) - throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName); + throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + client.whoami() + " table " + tableName); for (Entry<String,Integer> entry : WalkingSecurity.get(state, env).getAuthsMap().entrySet()) { if (auths.contains(entry.getKey().getBytes(UTF_8))) seen = seen - entry.getValue(); @@ -171,7 +170,7 @@ public class TableOp extends Test { BatchWriter writer = null; try { try { - writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1)); + writer = client.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1)); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); @@ -230,7 +229,7 @@ public class TableOp extends Test { throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName); return; } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { - if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami())) + if (WalkingSecurity.get(state, env).userPassTransient(client.whoami())) return; } throw new AccumuloException("Unexpected exception!", ae); @@ -252,7 +251,7 @@ public class TableOp extends Test { throw new AccumuloException("Table didn't exist when it should have: " + tableName, ase); return; } - AlterTable.renameTable(conn, state, env, tableName, tableName + "plus", tablePerm, tableExists); + AlterTable.renameTable(client, state, env, tableName, tableName + "plus", tablePerm, tableExists); break; case GRANT: diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java index 9e36c86..2cc0f4f 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java @@ -22,7 +22,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.security.SecurityErrorCode; -import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java index 924b8d9..9b080ea 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java @@ -25,7 +25,6 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.testing.core.TestProps; import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv; import org.apache.accumulo.testing.core.randomwalk.State; import org.apache.accumulo.testing.core.randomwalk.Test; @@ -36,16 +35,10 @@ public class MapRedVerify extends Test { @Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { - String[] args = new String[6]; - args[0] = env.getAccumuloUserName(); - args[1] = env.getAccumuloPassword(); - if (null == args[1]) { - args[1] = env.getAccumuloKeytab(); - } - args[2] = state.getString("seqTableName"); - args[3] = env.getAccumuloInstance().getInstanceName(); - args[4] = env.getConfigProperty(TestProps.ZOOKEEPERS); - args[5] = args[2] + "_MR"; + String[] args = new String[3]; + args[0] = env.getClientPropsPath(); + args[1] = state.getString("seqTableName"); + args[2] = args[2] + "_MR"; if (ToolRunner.run(env.getHadoopConfiguration(), new MapRedVerifyTool(), args) != 0) { log.error("Failed to run map/red verify"); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java index e8dcca1..de952ee 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java @@ -19,20 +19,13 @@ package org.apache.accumulo.testing.core.randomwalk.sequential; import java.io.IOException; import java.util.Iterator; -import org.apache.accumulo.core.client.ClientConfiguration; -import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.client.admin.DelegationTokenConfig; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.ClientInfo; import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; -import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; -import org.apache.accumulo.core.client.security.tokens.KerberosToken; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.SystemPermission; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; @@ -40,7 +33,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,53 +87,14 @@ public class MapRedVerifyTool extends Configured implements Tool { return 1; } - ClientConfiguration clientConf = ClientConfiguration.loadDefault().withInstance(args[3]).withZkHosts(args[4]); + ClientInfo info = Accumulo.newClient().usingProperties(args[0]).info(); + AccumuloInputFormat.setClientInfo(job, info); + AccumuloInputFormat.setInputTableName(job, args[1]); - AccumuloInputFormat.setInputTableName(job, args[2]); - AccumuloInputFormat.setZooKeeperInstance(job, clientConf); - AccumuloOutputFormat.setDefaultTableName(job, args[5]); - AccumuloOutputFormat.setZooKeeperInstance(job, clientConf); + AccumuloOutputFormat.setClientInfo(job, info); + AccumuloOutputFormat.setDefaultTableName(job, args[2]); job.setInputFormatClass(AccumuloInputFormat.class); - if (Boolean.parseBoolean(clientConf.get(ClientProperty.INSTANCE_RPC_SASL_ENABLED))) { - // Better be logged in - KerberosToken token = new KerberosToken(); - try { - UserGroupInformation user = UserGroupInformation.getCurrentUser(); - if (!user.hasKerberosCredentials()) { - throw new IllegalStateException("Expected current user to have Kerberos credentials"); - } - - String newPrincipal = user.getUserName(); - - ZooKeeperInstance inst = new ZooKeeperInstance(clientConf); - Connector conn = inst.getConnector(newPrincipal, token); - - // Do the explicit check to see if the user has the permission - // to get a delegation token - if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) { - log.error(newPrincipal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name() - + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's" - + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials."); - throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token"); - } - - // Fetch a delegation token from Accumulo - AuthenticationToken dt = conn.securityOperations().getDelegationToken(new DelegationTokenConfig()); - - // Set the delegation token instead of the kerberos token - AccumuloInputFormat.setConnectorInfo(job, newPrincipal, dt); - AccumuloOutputFormat.setConnectorInfo(job, newPrincipal, dt); - } catch (Exception e) { - final String msg = "Failed to acquire DelegationToken for use with MapReduce"; - log.error(msg, e); - throw new RuntimeException(msg, e); - } - } else { - AccumuloInputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1])); - AccumuloOutputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1])); - } - job.setMapperClass(SeqMapClass.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(IntWritable.class); diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java index 0514f05..3fe34a0 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java @@ -19,7 +19,6 @@ package org.apache.accumulo.testing.core.randomwalk.sequential; import java.net.InetAddress; import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableExistsException; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java index b520437..108dda1 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java @@ -36,7 +36,6 @@ import org.apache.accumulo.core.data.ColumnUpdate; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.testing.core.randomwalk.RandWalkEnv; import org.apache.accumulo.testing.core.randomwalk.State; import org.apache.accumulo.testing.core.randomwalk.Test; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java index 65b7a2b..708658e 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java @@ -23,7 +23,6 @@ import java.util.Random; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java index d27afa5..1cd4739 100644 --- a/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java +++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java @@ -18,7 +18,6 @@ package org.apache.accumulo.testing.core.stress; import org.apache.accumulo.core.cli.BatchWriterOpts; import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; diff --git a/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java index 02e7409..a8da367 100644 --- a/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java +++ b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java @@ -19,15 +19,12 @@ package org.apache.accumulo.testing.core.randomwalk; import static org.apache.accumulo.core.conf.Property.TSERV_NATIVEMAP_ENABLED; import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE; -import java.util.Properties; - -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.testing.core.randomwalk.concurrent.Replication; import org.apache.hadoop.conf.Configuration; +import org.easymock.EasyMock; import org.junit.Test; public class ReplicationRandomWalkIT extends ConfigurableMacBase { @@ -43,23 +40,13 @@ public class ReplicationRandomWalkIT extends ConfigurableMacBase { public void runReplicationRandomWalkStep() throws Exception { Replication r = new Replication(); - RandWalkEnv env = new RandWalkEnv(new Properties()) { - @Override - public String getAccumuloUserName() { - return "root"; - } - - @Override - public String getAccumuloPassword() { - return ROOT_PASSWORD; - } - - @Override - public Connector getAccumuloConnector() throws AccumuloException, AccumuloSecurityException { - return Connector.from(ReplicationRandomWalkIT.this.getClient()); - } + RandWalkEnv env = EasyMock.createMock(RandWalkEnv.class); + EasyMock.expect(env.getAccumuloUserName()).andReturn("root").anyTimes(); + EasyMock.expect(env.getAccumuloPassword()).andReturn(ROOT_PASSWORD).anyTimes(); + EasyMock.expect(env.getAccumuloClient()).andReturn(this.getClient()).anyTimes(); + EasyMock.expect(env.getAccumuloConnector()).andReturn(Connector.from(this.getClient())).anyTimes(); + EasyMock.replay(env); - }; r.visit(null, env, null); } diff --git a/pom.xml b/pom.xml index 69f7097..0de1eed 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ <properties> <accumulo.version>2.0.0-SNAPSHOT</accumulo.version> - <hadoop.version>2.6.4</hadoop.version> + <hadoop.version>3.0.3</hadoop.version> <zookeeper.version>3.4.9</zookeeper.version> <slf4j.version>1.7.21</slf4j.version> <twill.version>0.12.1</twill.version> @@ -53,12 +53,12 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>14.0.1</version> + <version>26.0-jre</version> </dependency> <dependency> <groupId>com.beust</groupId> <artifactId>jcommander</artifactId> - <version>1.48</version> + <version>1.72</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -73,7 +73,7 @@ <dependency> <groupId>commons-configuration</groupId> <artifactId>commons-configuration</artifactId> - <version>1.6</version> + <version>1.10</version> </dependency> <dependency> <groupId>org.apache.accumulo</groupId> diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java index 30b2191..56fe6f3 100644 --- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java +++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java @@ -23,11 +23,8 @@ import com.google.common.base.Preconditions; import org.apache.accumulo.testing.core.TestEnv; import org.apache.accumulo.testing.core.TestProps; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.twill.api.ResourceReport; import org.apache.twill.api.ResourceSpecification; import org.apache.twill.api.TwillApplication; -import org.apache.twill.api.TwillController; -import org.apache.twill.api.TwillRunResources; import org.apache.twill.api.TwillRunnerService; import org.apache.twill.api.TwillSpecification; import org.apache.twill.ext.BundledJarRunnable; @@ -39,7 +36,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Properties; @@ -69,12 +65,13 @@ public class YarnAccumuloTestRunner { .setMemory(memory, ResourceSpecification.SizeUnit.MEGA).setInstances(opts.numContainers).build(); File jarFile = new File(opts.jarPath); + File clientProps = new File(opts.clientProps); File testProps = new File(opts.testProps); File log4jProps = new File(opts.logProps); return TwillSpecification.Builder.with().setName(opts.testName).withRunnable().add(RUNNABLE_ID, new BundledJarRunnable(), resourceSpec).withLocalFiles() - .add(jarFile.getName(), jarFile.toURI(), false).add(testProps.getName(), testProps.toURI()).add(log4jProps.getName(), log4jProps.toURI()).apply() - .anyOrder().build(); + .add(jarFile.getName(), jarFile.toURI(), false).add(testProps.getName(), testProps.toURI()).add(clientProps.getName(), clientProps.toURI()) + .add(log4jProps.getName(), log4jProps.toURI()).apply().anyOrder().build(); } } @@ -92,6 +89,9 @@ public class YarnAccumuloTestRunner { @Parameter(names = {"--main", "-m"}, required = true, description = "Main class") String mainClass; + @Parameter(names = {"--clientProps", "-c"}, required = true, description = "Accumulo client properties path") + String clientProps; + @Parameter(names = {"--testProps", "-p"}, required = true, description = "Test properties path") String testProps; @@ -108,15 +108,6 @@ public class YarnAccumuloTestRunner { Preconditions.checkState(f.canRead()); } - private static int getNumRunning(TwillController controller) { - ResourceReport report = controller.getResourceReport(); - if (report == null) { - return 0; - } - Collection<TwillRunResources> resources = report.getRunnableResources(RUNNABLE_ID); - return resources == null ? 0 : resources.size(); - } - public static void main(String[] args) throws Exception { TestRunnerOpts opts = new TestRunnerOpts(); @@ -124,6 +115,7 @@ public class YarnAccumuloTestRunner { verifyPath(opts.jarPath); verifyPath(opts.testProps); + verifyPath(opts.clientProps); verifyPath(opts.logProps); String jarFileName = Paths.get(opts.jarPath).getFileName().toString(); @@ -131,26 +123,21 @@ public class YarnAccumuloTestRunner { String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new); BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(jarFileName, "/lib", opts.mainClass, mainArgs); - - Properties props = TestProps.loadFromFile(opts.testProps); - TestEnv env = new TestEnv(props); + TestEnv env = new TestEnv(opts.testProps, opts.clientProps); YarnConfiguration yarnConfig = new YarnConfiguration(env.getHadoopConfiguration()); - - TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getZookeepers()); + TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getInfo().getZooKeepers()); twillRunner.start(); - TwillController controller = twillRunner.prepare(new YarnTestApp(opts, props)) - .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName()).withArguments("BundledJarRunnable", arguments.toArray()).start(); + twillRunner.prepare(new YarnTestApp(opts, env.getTestProperties())) + .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName()) + .withArguments("BundledJarRunnable", arguments.toArray()).start(); - int numRunning = getNumRunning(controller); - while (numRunning != opts.numContainers) { - LOG.info("{} of {} containers have started in YARN.", numRunning, opts.numContainers); - Thread.sleep(5000); - numRunning = getNumRunning(controller); - } + LOG.info("{} containers will start in YARN.", opts.numContainers); + LOG.info("Press Ctrl-C when these containers have started."); - LOG.info("{} of {} containers have started in YARN", numRunning, opts.numContainers); - LOG.info("{} application was successfully started in YARN", opts.testName); + while (true) { + Thread.sleep(1000); + } } }