[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
src/start/src/main/java/org/apache/accumulo/start/Main.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/62580184
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/62580184
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/62580184
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 625801848cbb93ac0240eecf09ce62f225ee0960
Parents: 7e106a2 1543d65
Author: Mike Drob md...@cloudera.com
Authored: Tue Feb 25 14:38:25 2014 -0500
Committer: Mike Drob md...@cloudera.com
Committed: Tue Feb 25 14:38:25 2014 -0500
--
.../java/org/apache/accumulo/start/Main.java| 30
1 file changed, 24 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/62580184/start/src/main/java/org/apache/accumulo/start/Main.java
--
diff --cc start/src/main/java/org/apache/accumulo/start/Main.java
index cbc90c6,000..29c47a7
mode 100644,00..100644
--- a/start/src/main/java/org/apache/accumulo/start/Main.java
+++ b/start/src/main/java/org/apache/accumulo/start/Main.java
@@@ -1,125 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start;
+
++import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
-
++
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+
+public class Main {
+
- public static void main(String[] args) throws Exception {
++ public static void main(String[] args) {
+Runnable r = null;
+
+try {
+ if (args.length == 0) {
+printUsage();
+System.exit(1);
+ }
+ final String argsToPass[] = new String[args.length - 1];
+ System.arraycopy(args, 1, argsToPass, 0, args.length - 1);
+
+
Thread.currentThread().setContextClassLoader(AccumuloClassLoader.getClassLoader());
+
+ Class? vfsClassLoader =
AccumuloClassLoader.getClassLoader().loadClass(org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader);
+
+ ClassLoader cl = (ClassLoader)
vfsClassLoader.getMethod(getClassLoader, new Class[] {}).invoke(null, new
Object[] {});
+
+ Class? runTMP = null;
+
+ Thread.currentThread().setContextClassLoader(cl);
+
+ if (args[0].equals(master)) {
+runTMP = cl.loadClass(org.apache.accumulo.server.master.Master);
+ } else if (args[0].equals(tserver)) {
+runTMP =
cl.loadClass(org.apache.accumulo.server.tabletserver.TabletServer);
+ } else if (args[0].equals(shell)) {
+runTMP = cl.loadClass(org.apache.accumulo.core.util.shell.Shell);
+ } else if (args[0].equals(init)) {
+runTMP = cl.loadClass(org.apache.accumulo.server.util.Initialize);
+ } else if (args[0].equals(admin)) {
+runTMP = cl.loadClass(org.apache.accumulo.server.util.Admin);
+ } else if (args[0].equals(gc)) {
+runTMP =
cl.loadClass(org.apache.accumulo.server.gc.SimpleGarbageCollector);
+ } else if (args[0].equals(monitor)) {
+runTMP = cl.loadClass(org.apache.accumulo.server.monitor.Monitor);
+ } else if (args[0].equals(tracer)) {
+runTMP = cl.loadClass(org.apache.accumulo.server.trace.TraceServer);
+ } else if (args[0].equals(proxy)) {
+runTMP = cl.loadClass(org.apache.accumulo.proxy.Proxy);
+ } else if (args[0].equals(classpath)) {
+vfsClassLoader.getMethod(printClassPath, new Class[]
{}).invoke(vfsClassLoader, new Object[] {});
+return;
+ } else if (args[0].equals(version)) {
+runTMP = cl.loadClass(org.apache.accumulo.core.Constants);
+System.out.println(runTMP.getField(VERSION).get(null));
+return;
+ } else if (args[0].equals(rfile-info)) {
+runTMP =
cl.loadClass(org.apache.accumulo.core.file.rfile.PrintInfo);
+ }
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
src/trace/src/main/java/org/apache/accumulo/cloudtrace/instrument/TraceProxy.java
src/trace/src/test/java/org/apache/accumulo/cloudtrace/instrument/TracerTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/41ce56a5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/41ce56a5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/41ce56a5
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 41ce56a51ac2f18506e996a60fba44a464f65529
Parents: a38b004 2829426
Author: Mike Drob md...@cloudera.com
Authored: Mon Feb 24 12:26:48 2014 -0500
Committer: Mike Drob md...@cloudera.com
Committed: Mon Feb 24 12:26:48 2014 -0500
--
.../accumulo/trace/instrument/TraceProxy.java | 37 ---
.../accumulo/trace/instrument/TracerTest.java | 38
2 files changed, 63 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/41ce56a5/trace/src/main/java/org/apache/accumulo/trace/instrument/TraceProxy.java
--
diff --cc
trace/src/main/java/org/apache/accumulo/trace/instrument/TraceProxy.java
index a7eba1c,000..cb93210
mode 100644,00..100644
--- a/trace/src/main/java/org/apache/accumulo/trace/instrument/TraceProxy.java
+++ b/trace/src/main/java/org/apache/accumulo/trace/instrument/TraceProxy.java
@@@ -1,59 -1,0 +1,72 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.trace.instrument;
+
+import java.lang.reflect.InvocationHandler;
++import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
++import org.apache.log4j.Logger;
++
+public class TraceProxy {
- // private static final Logger log = Logger.getLogger(TraceProxy.class);
-
++ private static final Logger log = Logger.getLogger(TraceProxy.class);
++
+ static final Sampler ALWAYS = new Sampler() {
+@Override
+public boolean next() {
+ return true;
+}
+ };
-
++
+ public static T T trace(T instance) {
+return trace(instance, ALWAYS);
+ }
-
++
+ @SuppressWarnings(unchecked)
+ public static T T trace(final T instance, final Sampler sampler) {
+InvocationHandler handler = new InvocationHandler() {
+ @Override
+ public Object invoke(Object obj, Method method, Object[] args) throws
Throwable {
- if (!sampler.next()) {
- return method.invoke(instance, args);
++Span span = null;
++if (sampler.next()) {
++ span = Trace.on(method.getName());
+}
- Span span = Trace.on(method.getName());
+try {
+ return method.invoke(instance, args);
- } catch (Throwable ex) {
- ex.printStackTrace();
- throw ex;
++ // Can throw RuntimeException, Error, or any checked exceptions of
the method.
++} catch (InvocationTargetException ite) {
++ Throwable cause = ite.getCause();
++ if (cause == null) {
++// This should never happen, but account for it anyway
++log.error(Invocation exception during trace with null cause: ,
ite);
++throw new RuntimeException(ite);
++ }
++ throw cause;
++} catch (IllegalAccessException e) {
++ throw new RuntimeException(e);
+} finally {
- span.stop();
++ if (span != null) {
++span.stop();
++ }
+}
+ }
+};
+return (T) Proxy.newProxyInstance(instance.getClass().getClassLoader(),
instance.getClass().getInterfaces(), handler);
+ }
-
++
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/41ce56a5/trace/src/test/java/org/apache/accumulo/trace/instrument/TracerTest.java
--
diff --cc
trace/src/test/java/org/apache/accumulo/trace/instrument/TracerTest.java
index
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
test/system/auto/TestUtils.py
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/44a76367
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/44a76367
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/44a76367
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 44a76367add9b886a8c94e36b3476ba01e28f524
Parents: 1a79677 7059c76
Author: Bill Havanki bhava...@cloudera.com
Authored: Thu Feb 20 16:54:23 2014 -0500
Committer: Bill Havanki bhava...@cloudera.com
Committed: Thu Feb 20 16:54:23 2014 -0500
--
.../accumulo/test/functional/RunTests.java | 50
test/system/auto/run.py | 34 -
2 files changed, 73 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/44a76367/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
--
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
index 0da0f61,000..0f4dd8e
mode 100644,00..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
@@@ -1,169 -1,0 +1,201 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
++import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
++import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.List;
++import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Runs the functional tests via map-reduce.
+ *
+ * First, be sure everything is compiled.
+ *
+ * Second, get a list of the tests you want to run:
+ *
+ * pre
+ * $ python test/system/auto/run.py -l tests
+ * /pre
+ *
+ * Put the list of tests into HDFS:
+ *
+ * pre
+ * $ hadoop fs -put tests /user/hadoop/tests
+ * /pre
+ *
+ * Run the map-reduce job:
+ *
+ * pre
+ * $ ./bin/accumulo accumulo.test.functional.RunTests --tests
/user/hadoop/tests --output /user/hadoop/results
+ * /pre
+ *
+ * Note that you will need to have some configuration in
conf/accumulo-site.xml (to locate zookeeper). The map-reduce jobs will not use
your local accumulo
+ * instance.
+ *
+ */
+public class RunTests extends Configured implements Tool {
+
+ static final public String JOB_NAME = Functional Test Runner;
+ private static final Logger log = Logger.getLogger(RunTests.class);
+
+ private Job job = null;
+
+ static class Opts extends Help {
+@Parameter(names=--tests, description=newline separated list of tests
to run, required=true)
+String testFile;
+@Parameter(names=--output, description=destination for the results of
tests in HDFS, required=true)
+String outputPath;
+ }
+
+ static public class TestMapper extends MapperLongWritable,Text,Text,Text {
+
++private static final String REDUCER_RESULT_START = : ;
++private static final int RRS_LEN = REDUCER_RESULT_START.length();
++private Text result = new Text();
++
++private static enum Outcome {
++
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Conflicts: test/system/auto/README Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f6c268e5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f6c268e5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f6c268e5 Branch: refs/heads/1.5.1-SNAPSHOT Commit: f6c268e56b6428fbf9bb4da347ffe9406f2d8044 Parents: 8828328 ac18b56 Author: Bill Havanki bhava...@cloudera.com Authored: Wed Feb 19 11:11:06 2014 -0500 Committer: Bill Havanki bhava...@cloudera.com Committed: Wed Feb 19 11:11:06 2014 -0500 -- test/system/auto/README | 94 ++-- 1 file changed, 65 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f6c268e5/test/system/auto/README -- diff --cc test/system/auto/README index 7d18e8b,45ed158..41f6159 --- a/test/system/auto/README +++ b/test/system/auto/README @@@ -53,6 -63,32 +63,32 @@@ cluster at your disposal, you can run t $ python test/system/auto/run.py -l tests $ hadoop fs -put tests /user/hadoop/tests - $ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests /user/hadoop/tests --output /user/hadoop/results - $ ./bin/accumulo org.apache.accumulo.server.test.functional.RunTests \ - /user/hadoop/tests /user/hadoop/results ++ $ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ ++ /user/hadoop/tests --output /user/hadoop/results + + The example above runs every test. You can trim the tests file to include + only the tests you wish to run. + + In some clusters, the user under which MR jobs run is different from the user + under which Accumulo is installed, and this can cause failures running the + tests. Various configuration and permission changes can be made to help the + tests run, including the following: + + * Opening up directory and file permissions on each cluster node so that the MR + user has the same read/write privileges as the Accumulo user. Adding the MR + user to a shared group is one easy way to accomplish this. Access is required + to the Accumulo installation, log, write-ahead log, and configuration + directories. + * Creating a user directory in HDFS, named after and owned by the MR user, + e.g., /user/mruser. + * Setting the ZOOKEEPER_HOME and HADOOP_CONF_DIR environment variables for the + MR user. These can be set using the mapred.child.env property in + mapred-site.xml, e.g.: + + property + namemapred.child.env/name + valueZOOKEEPER_HOME=/path/to/zookeeper,HADOOP_CONF_DIR=/path/to/hadoop/conf/value + /property - + Each functional test is run by a mapper, and so you can check the mapper logs + to see any error messages tests produce.
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT (-sours)
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT (-sours) Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/688d4e47 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/688d4e47 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/688d4e47 Branch: refs/heads/1.5.1-SNAPSHOT Commit: 688d4e476181bfd4d93f44fea0bfd663ea13da75 Parents: be26f79 9bff89f Author: Bill Havanki bhava...@cloudera.com Authored: Thu Jan 30 11:21:59 2014 -0500 Committer: Bill Havanki bhava...@cloudera.com Committed: Thu Jan 30 11:21:59 2014 -0500 -- --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT (-sours)
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT (-sours) Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/295643ab Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/295643ab Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/295643ab Branch: refs/heads/1.5.1-SNAPSHOT Commit: 295643ab039e813d35d00f8c7e248b4ca79d199c Parents: 8981ba0 6593a9f Author: Bill Havanki bhava...@cloudera.com Authored: Fri Jan 24 15:24:17 2014 -0500 Committer: Bill Havanki bhava...@cloudera.com Committed: Fri Jan 24 15:24:17 2014 -0500 -- --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bd67c465 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bd67c465 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bd67c465 Branch: refs/heads/1.5.1-SNAPSHOT Commit: bd67c465ffa17b4b063f123d1494f1d29c411f0a Parents: 13eb19c 06f8030 Author: Bill Havanki bhava...@cloudera.com Authored: Wed Jan 22 14:05:28 2014 -0500 Committer: Bill Havanki bhava...@cloudera.com Committed: Wed Jan 22 14:05:28 2014 -0500 -- test/system/randomwalk/README | 10 ++ 1 file changed, 10 insertions(+) --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
test/system/continuous/run-verify.sh
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7bef4048
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7bef4048
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7bef4048
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 7bef40489b3049d3baf66b210bd63123687e18a0
Parents: bd67c46 57f9b6c
Author: Josh Elser els...@apache.org
Authored: Wed Jan 22 16:59:26 2014 -0500
Committer: Josh Elser els...@apache.org
Committed: Wed Jan 22 16:59:26 2014 -0500
--
.../core/client/mapreduce/RangeInputSplit.java | 11 -
.../mapreduce/lib/util/ConfiguratorBase.java| 14 +---
.../test/continuous/ContinuousVerify.java | 22 ++
test/system/continuous/run-verify.sh| 24 +++-
4 files changed, 66 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bef4048/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
--
diff --cc
core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 561e7ac,000..592cde6
mode 100644,00..100644
---
a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -1,433 -1,0 +1,442 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
++import org.apache.accumulo.core.conf.AccumuloConfiguration;
++import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.log4j.Level;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map
Reduce jobs.
+ */
+public class RangeInputSplit extends InputSplit implements Writable {
+ private Range range;
+ private String[] locations;
+ private String table, instanceName, zooKeepers, principal;
+ private AuthenticationToken token;
+ private Boolean offline, mockInstance, isolatedScan, localIterators;
+ private Authorizations auths;
+ private SetPairText,Text fetchedColumns;
+ private ListIteratorSetting iterators;
+ private Level level;
+
+ public RangeInputSplit() {
+range = new Range();
+locations = new String[0];
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+this.range =
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/78c5f460 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/78c5f460 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/78c5f460 Branch: refs/heads/1.5.1-SNAPSHOT Commit: 78c5f460b1a7f5b2762c2dba7437892151b4d59b Parents: 0328838 e887f37 Author: Mike Drob md...@cloudera.com Authored: Tue Jan 21 10:30:19 2014 -0500 Committer: Mike Drob md...@cloudera.com Committed: Tue Jan 21 10:30:19 2014 -0500 -- test/system/bench/README | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/78c5f460/test/system/bench/README -- diff --cc test/system/bench/README index 03b1e97,06e174b..e5fe692 --- a/test/system/bench/README +++ b/test/system/bench/README @@@ -8,10 -8,11 +8,11 @@@ Syntax for running run.py Specifying a specific benchmark or set of benchmarks runs only those, while not specifying any runs all benchmarks. -l Lists the benchmarks that will be run --v run_speed can either be slow, medium or fast --s log_level is a number representing the verbosity of the debugging output: 10 is debug, 20 is info, 30 is warning, etc. +-s run_speed can either be slow, medium or fast +-v log_level is a number representing the verbosity of the debugging output: 10 is debug, 20 is info, 30 is warning, etc. -u user user to use when connecting with accumulo. If not set you will be prompted to input it. -p password password to use when connecting with accumulo. If not set you will be prompted to input it. + -z zookeepers comma delimited lit of zookeeper host:port pairs to use when connecting with accumulo. If not set you will be prompted to input it. -i instance instance to use when connecting with accumulo. If not set you will be prompted to input it. **
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/52ef7c6e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/52ef7c6e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/52ef7c6e Branch: refs/heads/1.5.1-SNAPSHOT Commit: 52ef7c6e00beb6cd501ee57978eecea3adca109c Parents: 75f27e5 7a677b5 Author: Bill Havanki bhava...@cloudera.com Authored: Fri Jan 17 12:12:18 2014 -0500 Committer: Bill Havanki bhava...@cloudera.com Committed: Fri Jan 17 12:12:18 2014 -0500 -- .../system/randomwalk/conf/modules/LongEach.xml | 50 .../randomwalk/conf/modules/ShortEach.xml | 50 2 files changed, 100 insertions(+) --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7088004c Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7088004c Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7088004c Branch: refs/heads/1.5.1-SNAPSHOT Commit: 7088004c81618a5a5c5b7784e9b47f2647680e76 Parents: 833ac35 d2a1fe7 Author: Mike Drob md...@cloudera.com Authored: Tue Jan 7 09:54:22 2014 -0800 Committer: Mike Drob md...@cloudera.com Committed: Tue Jan 7 09:54:22 2014 -0800 -- test/system/continuous/continuous-env.sh.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7088004c/test/system/continuous/continuous-env.sh.example --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
test/system/auto/simple/readwrite.py
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/89c99622
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/89c99622
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/89c99622
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 89c996229dc259fd6c4555d97e1da7b0d0f2e7a7
Parents: 20cc9f4 d605ebb
Author: Mike Drob md...@cloudera.com
Authored: Tue Dec 31 13:37:29 2013 -0800
Committer: Mike Drob md...@cloudera.com
Committed: Tue Dec 31 13:37:29 2013 -0800
--
test/system/auto/TestUtils.py | 8 ++--
test/system/auto/simple/readwrite.py| 5 ++---
test/system/auto/simple/zooCacheTest.py | 4 ++--
3 files changed, 10 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89c99622/test/system/auto/TestUtils.py
--
diff --cc test/system/auto/TestUtils.py
index 32dc4b8,3101cf9..68d2a9c
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@@ -47,7 -47,10 +47,8 @@@ if os.getenv('ACCUMULO_CONF_DIR')
else:
ACCUMULO_CONF_DIR = os.path.join(ACCUMULO_HOME, 'conf')
SITE = test- + ID
+ SITE_PATH = os.path.join(ACCUMULO_CONF_DIR, SITE)
-WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)
-
LOG_PROPERTIES= os.path.join(ACCUMULO_CONF_DIR, 'log4j.properties')
LOG_GENERIC = os.path.join(ACCUMULO_CONF_DIR, 'generic_logger.xml')
LOG_MONITOR = os.path.join(ACCUMULO_CONF_DIR, 'monitor_logger.xml')
@@@ -252,10 -259,10 +253,10 @@@ class TestUtilsMixin
def stop_accumulo(self, signal=signal.SIGHUP):
log.info('killing accumulo processes everywhere')
for host in self.hosts:
-self.pkill(host, 'org.apache.accumulo.start', signal)
+self.pkill(host, 'accumulo.config.file', signal)
def create_config_file(self, settings):
- fp = open(os.path.join(ACCUMULO_CONF_DIR, SITE), 'w')
+ fp = open(SITE_PATH, 'w')
fp.write('configuration\n')
settings = self.settings.copy()
settings.update({ 'instance.zookeeper.host': ZOOKEEPERS,
@@@ -442,10 -455,11 +446,10 @@@
['hadoop', 'fs', '-rmr', ACCUMULO_DIR]))
self.wait(self.runClassOn(self.masterHost(),
'org.apache.accumulo.server.util.DeleteZooInstance',
-[INSTANCE_NAME]))
- self.wait(self.runOn(self.masterHost(), ['rm', '-rf', WALOG]))
+['-i', INSTANCE_NAME]))
self.wait(self.runOn(self.masterHost(), ['rm', '-rf', ACCUMULO_HOME
+ '/logs/' + ID]))
self.clean_logging()
- os.unlink(os.path.join(ACCUMULO_HOME, 'conf', SITE))
+ self.clean_config_file()
def createTable(self, table, splitFile=None):
if splitFile :
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89c99622/test/system/auto/simple/readwrite.py
--
diff --cc test/system/auto/simple/readwrite.py
index 895daf5,13d2afa..d252677
--- a/test/system/auto/simple/readwrite.py
+++ b/test/system/auto/simple/readwrite.py
@@@ -20,7 -20,7 +20,7 @@@ import unittes
import time
import sys
- from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE,
INSTANCE_NAME, ROOT, ROOT_PASSWORD
-from TestUtils import TestUtilsMixin, FUZZ, SITE_PATH
++from TestUtils import TestUtilsMixin, FUZZ, SITE_PATH, INSTANCE_NAME, ROOT,
ROOT_PASSWORD
log = logging.getLogger('test.auto')
http://git-wip-us.apache.org/repos/asf/accumulo/blob/89c99622/test/system/auto/simple/zooCacheTest.py
--
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/144d9d5b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/144d9d5b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/144d9d5b
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 144d9d5bfd3f249cdabfb8a7f5d60543ff6cc6c1
Parents: 3458bfa bec36bc
Author: Eric Newton eric.new...@gmail.com
Authored: Tue Dec 17 15:03:37 2013 -0500
Committer: Eric Newton eric.new...@gmail.com
Committed: Tue Dec 17 15:03:37 2013 -0500
--
.../accumulo/server/master/balancer/DefaultLoadBalancer.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/144d9d5b/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
--
diff --cc
server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
index 9b88d74,000..1fcab46
mode 100644,00..100644
---
a/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
+++
b/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
@@@ -1,318 -1,0 +1,319 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.log4j.Logger;
+
+public class DefaultLoadBalancer extends TabletBalancer {
+
+ private static final Logger log =
Logger.getLogger(DefaultLoadBalancer.class);
+
+ IteratorTServerInstance assignments;
+ // if tableToBalance is set, then only balance the given table
+ String tableToBalance = null;
+
+ public DefaultLoadBalancer() {
+
+ }
+
+ public DefaultLoadBalancer(String table) {
+tableToBalance = table;
+ }
+
+ ListTServerInstance randomize(SetTServerInstance locations) {
+ListTServerInstance result = new ArrayListTServerInstance(locations);
+Collections.shuffle(result);
+return result;
+ }
+
+ public TServerInstance
getAssignment(SortedMapTServerInstance,TabletServerStatus locations,
KeyExtent extent, TServerInstance last) {
+if (locations.size() == 0)
+ return null;
+
+if (last != null) {
+ // Maintain locality
- TServerInstance simple = new TServerInstance(last.getLocation(), );
++ String fakeSessionID = ;
++ TServerInstance simple = new TServerInstance(last.getLocation(),
fakeSessionID);
+ IteratorTServerInstance find =
locations.tailMap(simple).keySet().iterator();
+ if (find.hasNext()) {
+TServerInstance current = find.next();
+if (current.host().equals(last.host()))
+ return current;
+ }
+}
+
+// The strategy here is to walk through the locations and hand them back,
one at a time
+// Grab an iterator off of the set of options; use a new iterator if it
hands back something not in the current list.
+if (assignments == null || !assignments.hasNext())
+ assignments = randomize(locations.keySet()).iterator();
+TServerInstance result = assignments.next();
+if (!locations.containsKey(result)) {
+ assignments = null;
+ return randomize(locations.keySet()).iterator().next();
+}
+return result;
+ }
+
+ static class ServerCounts
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5fa1a75e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5fa1a75e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5fa1a75e Branch: refs/heads/1.5.1-SNAPSHOT Commit: 5fa1a75e393ce54fd8a675a949bfb820292b9dfa Parents: 480c964 72106a6 Author: Mike Drob md...@cloudera.com Authored: Thu Dec 12 17:49:47 2013 -0800 Committer: Mike Drob md...@cloudera.com Committed: Thu Dec 12 17:49:47 2013 -0800 -- --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/11ba435a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/11ba435a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/11ba435a Branch: refs/heads/1.5.1-SNAPSHOT Commit: 11ba435a4002ef44d18d0c646a3c7649943d7c80 Parents: ad9ee5e 51dd805 Author: Eric Newton eric.new...@gmail.com Authored: Tue Nov 26 11:04:02 2013 -0500 Committer: Eric Newton eric.new...@gmail.com Committed: Tue Nov 26 11:04:02 2013 -0500 -- --
[06/10] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8b5f2611
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8b5f2611
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8b5f2611
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 8b5f261160f9e9c786d2d6ae61eddcebede34c81
Parents: 7f403df d059d00
Author: Mike Drob md...@mdrob.com
Authored: Mon Nov 11 14:42:02 2013 -0500
Committer: Mike Drob md...@mdrob.com
Committed: Mon Nov 11 14:42:22 2013 -0500
--
.../accumulo/core/security/ColumnVisibility.java | 13 +++--
.../accumulo/core/security/ColumnVisibilityTest.java | 7 +++
2 files changed, 18 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8b5f2611/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
--
diff --cc
core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
index fe8128e,000..7d7daa2
mode 100644,00..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/ColumnVisibility.java
@@@ -1,499 -1,0 +1,508 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.security;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.util.BadArgumentException;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+
+/**
+ * Validate the column visibility is a valid expression and set the
visibility for a Mutation. See {@link
ColumnVisibility#ColumnVisibility(byte[])} for the
+ * definition of an expression.
+ */
+public class ColumnVisibility {
+
+ Node node = null;
+ private byte[] expression;
+
+ /**
+ * Accessor for the underlying byte string.
+ *
+ * @return byte array representation of a visibility expression
+ */
+ public byte[] getExpression() {
+return expression;
+ }
+
+ public static enum NodeType {
- TERM, OR, AND,
++EMPTY, TERM, OR, AND,
+ }
++
++ /**
++ * All empty nodes are equal and represent the same value.
++ */
++ private static final Node EMPTY_NODE = new Node(NodeType.EMPTY);
+
+ public static class Node {
+public final static ListNode EMPTY = Collections.emptyList();
+NodeType type;
+int start = 0;
+int end = 0;
+ListNode children = EMPTY;
+
+public Node(NodeType type) {
+ this.type = type;
+}
+
+public Node(int start, int end) {
+ this.type = NodeType.TERM;
+ this.start = start;
+ this.end = end;
+}
+
+public void add(Node child) {
+ if (children == EMPTY)
+children = new ArrayListNode();
+
+ children.add(child);
+}
+
+public NodeType getType() {
+ return type;
+}
+
+public ListNode getChildren() {
+ return children;
+}
+
+public int getTermStart() {
+ return start;
+}
+
+public int getTermEnd() {
+ return end;
+}
+
+public ByteSequence getTerm(byte expression[]) {
+ if (type != NodeType.TERM)
+throw new RuntimeException();
+
+ if (expression[start] == '') {
+// its a quoted term
+int qStart = start + 1;
+int qEnd = end - 1;
+
+return new ArrayByteSequence(expression, qStart, qEnd - qStart);
+ }
+ return new ArrayByteSequence(expression, start, end - start);
+}
+ }
+
+ public static class NodeComparator
