accumulo-testing git commit: ACCUMULO-4581 Fixing several bugs in agitator scripts

2017-02-17 Thread mwalch 
Repository: accumulo-testing
Updated Branches:
  refs/heads/master 28ca75a36 -> 7495ba604


ACCUMULO-4581 Fixing several bugs in agitator scripts

* Updated how perl scripts are called in bash
* Removed analyze-missing.pl as it relies on centralized logging
* Renamed accumulo home and conf dir variables names in agitator
  perl scripts


Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/7495ba60
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/7495ba60
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/7495ba60

Branch: refs/heads/master
Commit: 7495ba6048ead216e578e8b6cdcc86d4ae8b1b47
Parents: 28ca75a
Author: Mike Walch 
Authored: Wed Feb 1 15:53:47 2017 -0500
Committer: Mike Walch 
Committed: Mon Feb 6 15:55:26 2017 -0500

--
 bin/accumulo-testing |  28 ---
 conf/accumulo-testing-env.sh.example |  12 ++-
 libexec/analyze-missing.pl   | 127 --
 libexec/datanode-agitator.pl |  18 ++---
 libexec/master-agitator.pl   |  25 +++---
 libexec/tserver-agitator.pl  |  22 +++---
 6 files changed, 49 insertions(+), 183 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/bin/accumulo-testing
--
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index 7cf7206..84a1d40 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -116,6 +116,8 @@ function determine_mapred_main() {
 }
 
 function start_agitator() {
+  hash pssh 2>/dev/null || { echo >&2 "The agitator requires pssh to be 
installed. Aborting."; exit 1; }
+
   mkdir -p "${at_home}/logs"
   log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
   libexec="${at_home}/libexec"
@@ -132,17 +134,23 @@ function start_agitator() {
 su -c "$master_cmd >${master_log}.out 2>${master_log}.err" & -m - 
"$AGTR_ACCUMULO_USER"
 su -c "$tserver_cmd >${tserver_log}.out 2>${tserver_log}.err" & -m - 
"$AGTR_ACCUMULO_USER"
 su -c "$datanode_cmd >${datanode_log}.out 2>${datanode_log}.err" & -m - 
"$AGTR_HDFS_USER"
-  elif [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
-echo "Running master-agitator and tserver-agitator as $AGITATOR_USER 
Running datanode-agitator as $AGTR_HDFS_USER using sudo."
-$master_cmd > "${master_log}.out" 2> "${master_log}.err" &
-$tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
-sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> 
"${datanode_log}.err" &
   else
-echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER 
using sudo. Running datanode-agitator as $AGTR_HDFS_USER using sudo."
-# Not root, and not the accumulo user, hope you can sudo to it
-sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2> 
"${master_log}.err" &
-sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2> 
"${tserver_log}.err" &
-sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> 
"${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
+if [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
+  echo "Running master-agitator and tserver-agitator as $AGITATOR_USER"
+  $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
+  $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
+else 
+  echo "Running master-agitator and tserver-agitator as 
$AGTR_ACCUMULO_USER using sudo."
+  sudo -u "$AGTR_ACCUMULO_USER" $master_cmd > "${master_log}.out" 2> 
"${master_log}.err" &
+  sudo -u "$AGTR_ACCUMULO_USER" $tserver_cmd > "${tserver_log}.out" 2> 
"${tserver_log}.err" &
+fi
+if [[ $AGITATOR_USER == "$AGTR_HDFS_USER" ]]; then
+  echo "Running datanode-agitator as $AGITATOR_USER"
+  $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
+else
+  echo "Running datanode-agitator as $AGTR_HDFS_USER using sudo."
+  sudo -u "$AGTR_HDFS_USER" $datanode_cmd > "${datanode_log}.out" 2> 
"${datanode_log}.err" &
+fi
   fi
 
   if ${AGTR_HDFS:-false} ; then

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/7495ba60/conf/accumulo-testing-env.sh.example
--
diff --git a/conf/accumulo-testing-env.sh.example 
b/conf/accumulo-testing-env.sh.example
index 3f71463..e03c973 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -13,20 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Hadoop
-# ==
+# General
+# ===
 test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
-test -z

accumulo-examples git commit: ACCUMULO-4528 tool.sh is now 'accumulo-util hadoop-jar'

2017-02-17 Thread mwalch 
Repository: accumulo-examples
Updated Branches:
  refs/heads/master d10b96321 -> 71a84a1ef


ACCUMULO-4528 tool.sh is now 'accumulo-util hadoop-jar'

* Added note in README about copying jar to accumulo/lib/ext


Project: http://git-wip-us.apache.org/repos/asf/accumulo-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-examples/commit/71a84a1e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-examples/tree/71a84a1e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-examples/diff/71a84a1e

Branch: refs/heads/master
Commit: 71a84a1ef2326889a79991dd3ba1ae30d2688119
Parents: d10b963
Author: Mike Walch 
Authored: Fri Feb 17 11:36:28 2017 -0500
Committer: Mike Walch 
Committed: Fri Feb 17 15:21:16 2017 -0500

--
 README.md   | 12 
 docs/bulkIngest.md  |  2 +-
 docs/filedata.md|  2 +-
 docs/mapred.md  |  6 +++---
 docs/regex.md   |  2 +-
 docs/rowhash.md |  2 +-
 docs/tabletofile.md |  2 +-
 docs/terasort.md|  2 +-
 8 files changed, 17 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo-examples/blob/71a84a1e/README.md
--
diff --git a/README.md b/README.md
index 63a02d8..c4450eb 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,12 @@ Before running any of the examples, the following steps must 
be performed.
 cp examples.conf.template examples.conf
 nano examples.conf
 
-5. Each Accumulo example has its own documentation and instructions for 
running the example which
+5. The examples have some custom iterators that need to be executed by 
Accumulo tablet servers.
+   Make them available by copying the accumulo-examples.jar to Accumulo's 
`lib/ext` directory.
+
+cp target/accumulo-examples-X.Y.Z.jar /path/accumulo/lib/ext/
+
+6. Each Accumulo example has its own documentation and instructions for 
running the example which
are linked to below.
 
 When running the examples, remember the tips below:
@@ -50,9 +55,8 @@ When running the examples, remember the tips below:
   The `runex` command is a simple wrapper around the Maven Exec plugin.
 * Commands intended to be run in bash are prefixed by '$' and should be run 
from the root of this
   repository.
-* Several examples use the `accumulo` and `tool.sh` commands which are 
expected to be on your 
-  `PATH`. These commands are found in the `bin/` and `contrib/` directories of 
your Accumulo
-  installation.
+* Several examples use the `accumulo` and `accumulo-util` commands which are 
expected to be on your 
+  `PATH`. These commands are found in the `bin/` directory of your Accumulo 
installation.
 * Commands intended to be run in the Accumulo shell are prefixed by '>'.
 
 ## Available Examples

http://git-wip-us.apache.org/repos/asf/accumulo-examples/blob/71a84a1e/docs/bulkIngest.md
--
diff --git a/docs/bulkIngest.md b/docs/bulkIngest.md
index 614bde4..22bf07c 100644
--- a/docs/bulkIngest.md
+++ b/docs/bulkIngest.md
@@ -27,7 +27,7 @@ accumulo. Then we verify the 1000 rows are in accumulo.
 $ ARGS="-i instance -z zookeepers -u username -p password"
 $ accumulo $PKG.SetupTable $ARGS -t test_bulk row_0333 row_0666
 $ accumulo $PKG.GenerateTestData --start-row 0 --count 1000 --output 
bulk/test_1.txt
-$ tool.sh target/accumulo-examples.jar $PKG.BulkIngestExample $ARGS -t 
test_bulk --inputDir bulk --workDir tmp/bulkWork
+$ accumulo-util hadoop-jar target/accumulo-examples.jar 
$PKG.BulkIngestExample $ARGS -t test_bulk --inputDir bulk --workDir tmp/bulkWork
 $ accumulo $PKG.VerifyIngest $ARGS -t test_bulk --start-row 0 --count 1000
 
 For a high level discussion of bulk ingest, see the docs dir.

http://git-wip-us.apache.org/repos/asf/accumulo-examples/blob/71a84a1e/docs/filedata.md
--
diff --git a/docs/filedata.md b/docs/filedata.md
index 84311d2..aacd86e 100644
--- a/docs/filedata.md
+++ b/docs/filedata.md
@@ -40,7 +40,7 @@ Open the accumulo shell and look at the data. The row is the 
MD5 hash of the fil
 
 Run the CharacterHistogram MapReduce to add some information about the file.
 
-$ tool.sh target/accumulo-examples.jar 
org.apache.accumulo.examples.filedata.CharacterHistogram -c ./examples.conf -t 
dataTable --auths exampleVis --vis exampleVis
+$ accumulo-util hadoop-jar target/accumulo-examples.jar 
org.apache.accumulo.examples.filedata.CharacterHistogram -c ./examples.conf -t 
dataTable --auths exampleVis --vis exampleVis
 
 Scan again to see the histogram stored in the 'info' column family.
 

http://git-wip-us.apache.org/repos/asf/accumulo-examples/blob/71a84a1e/docs/mapred.md

[3/4] accumulo git commit: ACCUMULO-4528 Accumulo scripts improvements

2017-02-17 Thread mwalch 
ACCUMULO-4528 Accumulo scripts improvements

* Accumulo now determines hostname in Java
* Minimized use of environment variables
* Consolidated scripts by moving code in service.sh and cluster.sh to
  accumulo-service and accumulo-cluster.
* Example host files (i.e master, tservers) are no longer included in
  tarball distribution. User can create them using 'accumulo-cluster
  create-config'
* Moved check-tservers from distribution to contrib/scripts in repo.
* NUMA and multiple tservers are no longer configured in scripts but
  could be NUMA commands could be added using ACCUMULO_JAVA_PREFIX env
  variable.
* Removed accumulo-watcher script as restarting services should not be
  handled by Accumulo scripts. Script coud still be used in future
  using ACCUMULO_JAVA_PREFIX.
* Made several updates to accumulo-env.sh
* Moved create-config and build-native from accumulo script to new
  accumulo-util script which now includes scripts from contrib dir


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9a3cc5c4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9a3cc5c4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9a3cc5c4

Branch: refs/heads/master
Commit: 9a3cc5c445293e8d4219cc066757a72a972804b4
Parents: 00b1535
Author: Mike Walch 
Authored: Thu Feb 2 12:15:17 2017 -0500
Committer: Mike Walch 
Committed: Fri Feb 17 13:47:06 2017 -0500

--
 INSTALL.md  |  42 +-
 assemble/bin/accumulo   | 610 +++--
 assemble/bin/accumulo-cluster   | 300 +++-
 assemble/bin/accumulo-service   |  95 ++-
 assemble/bin/accumulo-util  | 686 +++
 assemble/conf/examples/gc   |  16 -
 .../conf/examples/generic_logger.properties |   6 +-
 assemble/conf/examples/generic_logger.xml   |   6 +-
 assemble/conf/examples/masters  |  16 -
 assemble/conf/examples/monitor  |  16 -
 .../conf/examples/templates/accumulo-env.sh |  64 ++
 .../conf/examples/templates/accumulo-site.xml   | 175 +
 assemble/conf/examples/tracers  |  16 -
 assemble/conf/examples/tservers |  16 -
 assemble/contrib/bootstrap-hdfs.sh  |  91 ---
 assemble/contrib/check-tservers | 199 --
 assemble/contrib/gen-monitor-cert.sh|  85 ---
 assemble/contrib/tool.sh|  93 ---
 assemble/libexec/accumulo-watcher.sh| 141 
 assemble/libexec/cluster.sh | 325 -
 assemble/libexec/load-env.sh| 155 -
 assemble/libexec/service.sh | 239 ---
 assemble/libexec/templates/accumulo-env.sh  |  90 ---
 assemble/libexec/templates/accumulo-site.xml| 175 -
 assemble/pom.xml|   7 +-
 assemble/src/main/assemblies/component.xml  |  50 +-
 .../main/scripts/generate-example-configs.sh|   2 +-
 contrib/scripts/check-tservers.py   | 199 ++
 .../accumulo/core/conf/ConfigurationDocGen.java |   2 +-
 .../org/apache/accumulo/core/conf/Property.java |   2 +-
 .../main/asciidoc/chapters/administration.txt   | 187 ++---
 docs/src/main/asciidoc/chapters/clients.txt |   4 +-
 docs/src/main/asciidoc/chapters/kerberos.txt|  14 +-
 docs/src/main/asciidoc/chapters/replication.txt |   4 +-
 docs/src/main/asciidoc/chapters/ssl.txt |   4 +-
 .../main/asciidoc/chapters/troubleshooting.txt  |   7 +-
 .../apache/accumulo/server/init/Initialize.java |   2 +-
 .../java/org/apache/accumulo/start/Main.java|   2 -
 38 files changed, 1691 insertions(+), 2452 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/INSTALL.md
--
diff --git a/INSTALL.md b/INSTALL.md
index b614991..6e924c4 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -25,11 +25,13 @@ source code.  Unpack as follows.
 tar xzf /accumulo-X.Y.Z-bin.tar.gz
 cd accumulo-X.Y.Z
 
-There are three scripts in the the `bin/` directory that are used to manage 
Accumulo:
+There are four scripts in the `bin` directory of the tarball distribution that 
are used
+to manage Accumulo:
 
 1. `accumulo` - Runs Accumulo command-line tools and starts Accumulo processes
 2. `accumulo-service` - Runs Accumulo processes as services
 3. `accumulo-cluster` - Manages Accumulo cluster on a single node or several 
nodes
+4. `accumulo-util` - Accumulo utilities for creating configuration, native 
libraries, etc.
 
 These scripts will be used in the remaining instructions to configure and run 
Accumulo.
 For convenience, consider adding `accumulo-X.Y.Z/bin/`

[1/4] accumulo git commit: ACCUMULO-4528 Accumulo scripts improvements

2017-02-17 Thread mwalch 
Repository: accumulo
Updated Branches:
  refs/heads/master e2247ead5 -> 6494a829b


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/docs/src/main/asciidoc/chapters/administration.txt
--
diff --git a/docs/src/main/asciidoc/chapters/administration.txt 
b/docs/src/main/asciidoc/chapters/administration.txt
index 1478543..f2ed57a 100644
--- a/docs/src/main/asciidoc/chapters/administration.txt
+++ b/docs/src/main/asciidoc/chapters/administration.txt
@@ -40,7 +40,7 @@ network bandwidth must be available between any two machines.
 
 In addition to needing access to ports associated with HDFS and ZooKeeper, 
Accumulo will
 use the following default ports. Please make sure that they are open, or change
-their value in conf/accumulo-site.xml.
+their value in accumulo-site.xml.
 
 .Accumulo default ports
 [width="75%",cols=">,^2,^2"]
@@ -80,24 +80,15 @@ sufficient space:
 Repeat this step on each machine in your cluster. Typically, the same 
++
 is chosen for all machines in the cluster.
 
-There are three scripts in the the `bin/` directory that are used to manage 
Accumulo:
+There are four scripts in the `bin/` directory that are used to manage 
Accumulo:
 
 1. `accumulo` - Runs Accumulo command-line tools and starts Accumulo processes
 2. `accumulo-service` - Runs Accumulo processes as services
 3. `accumulo-cluster` - Manages Accumulo cluster on a single node or several 
nodes
+4. `accumulo-util` - Accumulo utilities for creating configuration, native 
libraries, etc.
 
 These scripts will be used in the remaining instructions to configure and run 
Accumulo.
 
-Add the following to your `~/.bashrc` if you want to be able to execute these 
scripts
-from any directory:
-
-  export PATH=/path/to/accumulo-X.Y.Z/bin:$PATH
-
-Source your `~/.bashrc` for the changes to take effect and verify:
-
-  source ~/.bashrc
-  which accumulo
-
 === Dependencies
 
 Accumulo requires HDFS and ZooKeeper to be configured and running
@@ -108,32 +99,37 @@ sync, which can cause problems with automatically 
timestamped data.
 
 === Configuration
 
-Accumulo is configured by editing several Shell and XML files found in
-+$ACCUMULO_CONF_DIR+. The structure closely resembles Hadoop's configuration
-files.
+The Accumulo tarball contains a +conf/+ directory where Accumulo looks for 
configuration. If you
+installed Accumulo using downstream packaging, the +conf/+ could be something 
else like +/etc/accumulo/+.
 
-Logging is primarily controlled using the log4j configuration files,
-+generic_logger.xml+ and +monitor_logger.xml+ (or their corresponding
-+.properties+ version if the +.xml+ version is missing). The generic logger is
-used for most server types, and is typically configured to send logs to the
-monitor, as well as log files. The monitor logger is used by the monitor, and
-is typically configured to log only errors the monitor itself generates,
-rather than all the logs that it receives from other server types.
+Before starting Accumulo, the configuration files +accumulo-env.sh+ and 
+accumulo-site.xml+ must exist in
++conf/+ and be properly configured.  You can either copy these files from 
+conf/examples+ or run
++accumulo-util create-config+ to generate these files. If you are using 
+accumulo-cluster+ to launch
+a cluster, the `conf/` directory must also contain hosts file for Accumulo 
services (i.e +gc+, +masters+,
++monitor+, +tservers+, +tracers+). You can either create these files manually 
or run 
++accumulo-cluster create-config+.
 
- Edit conf/accumulo-env.sh
+Additional optional configuration files can be copied from `conf/examples` to 
`conf/`. For example,
+logging is primarily controlled using the log4j configuration files, 
+generic_logger.xml+ and
++monitor_logger.xml+ (or their corresponding +.properties+ version if the 
+.xml+ version is missing).
+The generic logger is used for most server types, and is typically configured 
to send logs to the
+monitor, as well as log files. The monitor logger is used by the monitor, and 
is typically configured
+to log only errors the monitor itself generates, rather than all the logs that 
it receives from other
+server types.
+
+ Configure accumulo-env.sh
 
 Accumulo needs to know where to find the software it depends on. Edit 
accumulo-env.sh
 and specify the following:
 
-. Enter your system's Java home for +$JAVA_HOME+
 . Enter the location of Hadoop for +$HADOOP_PREFIX+
-. Choose a location for Accumulo logs and enter it for +$ACCUMULO_LOG_DIR+
 . Enter the location of ZooKeeper for +$ZOOKEEPER_HOME+
+. Optionally, choose a different location for Accumulo logs using 
+$ACCUMULO_LOG_DIR+
 
-By default Accumulo TabletServers are set to use 1GB of memory. You may change
-this by altering the value of +$ACCUMULO_TSERVER_OPTS+. Note the syntax is 
that of
-the Java JVM command line options. This value should be less than the physical
-memory of the machines running TabletServers.
+You

[4/4] accumulo git commit: Merge branch 'script-refactor'

2017-02-17 Thread mwalch 
Merge branch 'script-refactor'


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6494a829
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6494a829
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6494a829

Branch: refs/heads/master
Commit: 6494a829b9560b3b38359641f25ecdcf3aac9a75
Parents: e2247ea 9a3cc5c
Author: Mike Walch 
Authored: Fri Feb 17 14:05:34 2017 -0500
Committer: Mike Walch 
Committed: Fri Feb 17 14:05:34 2017 -0500

--
 INSTALL.md  |  42 +-
 assemble/bin/accumulo   | 610 +++--
 assemble/bin/accumulo-cluster   | 300 +++-
 assemble/bin/accumulo-service   |  95 ++-
 assemble/bin/accumulo-util  | 686 +++
 assemble/conf/examples/gc   |  16 -
 .../conf/examples/generic_logger.properties |   6 +-
 assemble/conf/examples/generic_logger.xml   |   6 +-
 assemble/conf/examples/masters  |  16 -
 assemble/conf/examples/monitor  |  16 -
 .../conf/examples/templates/accumulo-env.sh |  64 ++
 .../conf/examples/templates/accumulo-site.xml   | 175 +
 assemble/conf/examples/tracers  |  16 -
 assemble/conf/examples/tservers |  16 -
 assemble/contrib/bootstrap-hdfs.sh  |  91 ---
 assemble/contrib/check-tservers | 199 --
 assemble/contrib/gen-monitor-cert.sh|  85 ---
 assemble/contrib/tool.sh|  93 ---
 assemble/libexec/accumulo-watcher.sh| 141 
 assemble/libexec/cluster.sh | 325 -
 assemble/libexec/load-env.sh| 155 -
 assemble/libexec/service.sh | 239 ---
 assemble/libexec/templates/accumulo-env.sh  |  90 ---
 assemble/libexec/templates/accumulo-site.xml| 175 -
 assemble/pom.xml|   7 +-
 assemble/src/main/assemblies/component.xml  |  50 +-
 .../main/scripts/generate-example-configs.sh|   2 +-
 contrib/scripts/check-tservers.py   | 199 ++
 .../accumulo/core/conf/ConfigurationDocGen.java |   2 +-
 .../org/apache/accumulo/core/conf/Property.java |   2 +-
 .../main/asciidoc/chapters/administration.txt   | 187 ++---
 docs/src/main/asciidoc/chapters/clients.txt |   4 +-
 docs/src/main/asciidoc/chapters/kerberos.txt|  14 +-
 docs/src/main/asciidoc/chapters/replication.txt |   4 +-
 docs/src/main/asciidoc/chapters/ssl.txt |   4 +-
 .../main/asciidoc/chapters/troubleshooting.txt  |   7 +-
 .../apache/accumulo/server/init/Initialize.java |   2 +-
 .../java/org/apache/accumulo/start/Main.java|   2 -
 38 files changed, 1691 insertions(+), 2452 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/6494a829/docs/src/main/asciidoc/chapters/administration.txt
--
diff --cc docs/src/main/asciidoc/chapters/administration.txt
index bfddb31,f2ed57a..ad6cf11
--- a/docs/src/main/asciidoc/chapters/administration.txt
+++ b/docs/src/main/asciidoc/chapters/administration.txt
@@@ -1178,24 -1129,25 +1129,16 @@@ are true interruptions to availability 
  Several of these conditions become more common in VM environments, where they
  can be exacerbated by resource constraints and clock drift.
  
- Accumulo includes a mechanism to limit the impact of the false positives known
- as the <>. The watcher monitors Accumulo processes and will restart
- them when they fail for certain reasons. The watcher can be configured within
- the accumulo-env.sh file inside of Accumulo's configuration directory. We
- recommend using the watcher to monitor Accumulo processes, as it will restore
- the system to full capacity without administrator interaction after many of 
the
- common failure modes.
- 
   Tested Versions
 -Another large consideration for Accumulo stability is to use versions of
 -software that have been tested together in a VM environment. Any cluster of
 -processes that have not been tested together are likely to expose running
 -conditions that vary from the environments individually tested in the various
 -components. For example, Accumulo's use of HDFS includes many short block
 -reads, which differs from the more common full file read used in most
 -map/reduce applications. We have found that certain versions of Accumulo and
 -Hadoop will include stability bugs that greatly affect overall stability. In
 -our testing, Accumulo 1.6.2, Hadoop 2.6.0, and Zookeeper 3.4.6 resulted in a
 -stable VM clusters that did not fail a month of testing, while Accumulo 1.6.1,
 -Hadoop 2.5.1, and Zookeeper 3.4.5 had a mean

[2/4] accumulo git commit: ACCUMULO-4528 Accumulo scripts improvements

2017-02-17 Thread mwalch 
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/conf/examples/templates/accumulo-site.xml
--
diff --git a/assemble/conf/examples/templates/accumulo-site.xml 
b/assemble/conf/examples/templates/accumulo-site.xml
new file mode 100644
index 000..a1f4153
--- /dev/null
+++ b/assemble/conf/examples/templates/accumulo-site.xml
@@ -0,0 +1,175 @@
+
+
+
+
+
+  
+  
+instance.volumes
+
+comma separated list of URIs for volumes. example: 
hdfs://localhost:9000/accumulo
+  
+
+  
+instance.zookeeper.host
+localhost:2181
+comma separated list of zookeeper servers
+  
+
+  
+instance.secret
+DEFAULT
+A secret unique to a given instance that all servers must 
know in order to communicate with one another.
+  Change it before initialization. To
+  change it later use ./bin/accumulo 
org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new 
[newpasswd],
+  and then update this file.
+
+  
+
+  
+tserver.memory.maps.max
+${memMapMax}
+  
+
+  
+tserver.memory.maps.native.enabled
+${nativeEnabled}
+  
+
+  
+tserver.cache.data.size
+${cacheDataSize}
+  
+
+  
+tserver.cache.index.size
+${cacheIndexSize}
+  
+
+  
+trace.token.property.password
+
+secret
+  
+
+  
+  
+instance.rpc.sasl.enabled
+true
+  
+
+  
+general.kerberos.keytab
+${keytab}
+  
+
+  
+general.kerberos.principal
+${principal}
+  
+
+  
+trace.token.type
+
org.apache.accumulo.core.client.security.tokens.KerberosToken
+  
+
+  
+instance.security.authenticator
+
org.apache.accumulo.server.security.handler.KerberosAuthenticator
+  
+
+  
+instance.security.authorizor
+
org.apache.accumulo.server.security.handler.KerberosAuthorizor
+  
+
+  
+instance.security.permissionHandler
+
org.apache.accumulo.server.security.handler.KerberosPermissionHandler
+  
+  
+
+  
+trace.user
+${traceUser}
+  
+
+  
+tserver.sort.buffer.size
+${sortBufferSize}
+  
+
+  
+tserver.walog.max.size
+${waLogMaxSize}
+  
+${mvnProjBaseDir}
+  
+general.classpaths
+
+
+  
+  $ACCUMULO_HOME/lib/accumulo-server.jar,
+  $ACCUMULO_HOME/lib/accumulo-core.jar,
+  $ACCUMULO_HOME/lib/accumulo-start.jar,
+  $ACCUMULO_HOME/lib/accumulo-fate.jar,
+  $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+  $ACCUMULO_HOME/lib/[^.].*.jar,
+  
+  $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+  
+  $HADOOP_CONF_DIR,
+  
+  $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar,
+  $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar,
+  $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar,
+  $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar,
+  $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar,
+  $HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar,
+  
+  
+  /usr/lib/hadoop/[^.].*.jar,
+  /usr/lib/hadoop/lib/[^.].*.jar,
+  /usr/lib/hadoop-hdfs/[^.].*.jar,
+  /usr/lib/hadoop-mapreduce/[^.].*.jar,
+  /usr/lib/hadoop-yarn/[^.].*.jar,
+  /usr/lib/hadoop-yarn/lib/jersey.*.jar,
+  
+  
+  /usr/hdp/current/hadoop-client/[^.].*.jar,
+  /usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar,
+  /usr/hdp/current/hadoop-hdfs-client/[^.].*.jar,
+  /usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar,
+  /usr/hdp/current/hadoop-yarn-client/[^.].*.jar,
+  /usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar,
+  /usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar
+  
+  
+  /usr/iop/current/hadoop-client/[^.].*.jar,
+  /usr/iop/current/hadoop-client/lib/(?!slf4j)[^.].*.jar,
+  /usr/iop/current/hadoop-hdfs-client/[^.].*.jar,
+  /usr/iop/current/hadoop-mapreduce-client/[^.].*.jar,
+  /usr/iop/current/hadoop-yarn-client/[^.].*.jar,
+  /usr/iop/current/hadoop-yarn-client/lib/jersey.*.jar,
+  /usr/iop/current/hive-client/lib/hive-accumulo-handler.jar
+  
+
+Classpaths that accumulo checks for updates and class 
files.
+  
+

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a3cc5c4/assemble/conf/examples/tracers
--
diff --git a/assemble/conf/examples/tracers b/assemble/conf/examples/tracers
deleted file mode 100644
index 63fb8bb..000
--- a/assemble/conf/examples/tracers
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed

[2/3] accumulo-wikisearch git commit: Use hadoop-client dep instead of hadoop-common

2017-02-17 Thread mmiller 
Use hadoop-client dep instead of hadoop-common


Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/fb4af6c2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/fb4af6c2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/fb4af6c2

Branch: refs/heads/master
Commit: fb4af6c2d50d061f33ae4341d37290b6b56bd1c4
Parents: eb573a5
Author: Mike Miller 
Authored: Thu Feb 16 15:52:21 2017 -0500
Committer: Mike Miller 
Committed: Fri Feb 17 10:38:49 2017 -0500

--
 pom.xml   |  2 +-
 query/pom.xml | 14 ++
 2 files changed, 3 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/fb4af6c2/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 08e9dbb..d62928a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,7 +68,7 @@

  
   org.apache.hadoop
-  hadoop-common
+  hadoop-client
   ${version.hadoop}
 
 

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/fb4af6c2/query/pom.xml
--
diff --git a/query/pom.xml b/query/pom.xml
index 76494eb..e022965 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -74,17 +74,7 @@
 
 
   org.apache.hadoop
-  hadoop-common
- 
-
-com.sun.jersey
-jersey-json
-
-
-javax.servlet
-servlet-api
-
-
+  hadoop-client
 
 
   javaee
@@ -156,7 +146,7 @@
 
   lib
   
-  
commons-io,commons-configuration,commons-lang,commons-codec,protobuf-java,libthrift,zookeeper,hadoop-common,commons-jexl,guava,kryo,asm,minlog,reflectasm,wikisearch-ingest,accumulo-core,accumulo-fate,accumulo-trace,htrace-core
+  
commons-io,commons-configuration,commons-lang,commons-codec,protobuf-java,libthrift,zookeeper,hadoop-client,commons-jexl,guava,kryo,asm,minlog,reflectasm,wikisearch-ingest,accumulo-core,accumulo-fate,accumulo-trace,htrace-core
   true

[1/3] accumulo-wikisearch git commit: Fixes to query for 1.8 Update

2017-02-17 Thread mmiller 
Repository: accumulo-wikisearch
Updated Branches:
  refs/heads/master 074fa7729 -> e51ad6a2f


Fixes to query for 1.8 Update


Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/eb573a51
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/eb573a51
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/eb573a51

Branch: refs/heads/master
Commit: eb573a5188c10fc82ddf911f4a0ab892860335d1
Parents: 074fa77
Author: Mike Miller 
Authored: Wed Jan 4 14:19:33 2017 -0500
Committer: Mike Miller 
Committed: Thu Feb 16 10:53:38 2017 -0500

--
 .gitignore  |   3 +-
 INSTALL.md  |  14 +-
 pom.xml |   3 +-
 query-war/src/main/webapp/WEB-INF/web.xml   |  10 --
 query-war/src/main/webapp/ui.html   | 131 +++
 query-war/src/main/webapp/ui.jsp| 131 ---
 query/pom.xml   |  29 +++-
 .../iterator/DefaultIteratorEnvironment.java|  57 ++--
 .../src/main/resources/META-INF/ejb-jar.xml.uno |  62 +
 9 files changed, 242 insertions(+), 198 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/eb573a51/.gitignore
--
diff --git a/.gitignore b/.gitignore
index 8a09ac0..9a97fb5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@
 .project
 .settings/
 .classpath
-
+wikipedia.xml
+ejb-jar.xml

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/eb573a51/INSTALL.md
--
diff --git a/INSTALL.md b/INSTALL.md
index 9a85105..f7cc175 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -49,9 +49,12 @@ Instructions for installing and running the Accumulo 
Wikisearch example.
  
 ### Prerequisites
 
-1. The query software was tested using JBoss AS 6. Install this unless you 
feel like messing with the installation.
-  - NOTE: Ran into a [bug] that did not allow an EJB3.1 war file. The 
workaround is to separate the RESTEasy servlet
-from the EJBs by creating an EJB jar and a WAR file.
+1. The query software was tested using JBoss AS 6. Install the JBoss distro 
and follow the instructions below
+   to build the EJB jar and WAR file required.
+  * To stop the JBoss warnings about WSDescriptorDeployer and 
JMSDescriptorDeployer, these deployers can be
+removed from 
`$JBOSS_HOME/server/default/deployers/jbossws.deployer/META-INF/stack-agnostic-jboss-beans.xml`
+1. Ensure that you have successfully run `mvn clean install` at the Wikisearch 
top level to install the jars
+   into your local maven repo before building the query package.

 ### Instructions
 
@@ -85,7 +88,7 @@ Instructions for installing and running the Accumulo 
Wikisearch example.
 
 1. At this point you should be able to open a browser and view the page:
 
-http://localhost:8080/accumulo-wikisearch/ui/ui.jsp
+http://localhost:8080/accumulo-wikisearch/ui.html
 
   You can issue the queries using this user interface or via the following 
REST urls:
 
@@ -98,6 +101,9 @@ Instructions for installing and running the Accumulo 
Wikisearch example.
into the search box at ui.jsp, and the auths parameter is a 
comma-separated list of wikis that you want to search (i.e.
enwiki,frwiki,dewiki, etc. Or you can use all) 

+  - NOTE: Ran into a [bug] that did not allow an EJB3.1 war file. The 
workaround is to separate the RESTEasy servlet
+from the EJBs by creating an EJB jar and a WAR file.
+
 [ejb-jar.xml.example]: query/src/main/resources/META-INF/ejb-jar.xml.example
 [dump-files]: http://dumps.wikimedia.org/backup-index.html
 [wikipedia.xml.example]: ingest/conf/wikipedia.xml.example

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/eb573a51/pom.xml
--
diff --git a/pom.xml b/pom.xml
index bebc1cf..08e9dbb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,6 +45,7 @@
 1.0.1.Final
 11.0.2
 2.6.4
+3.1.0-incubating
 2.1.0.GA
 4.11
 1.04
@@ -67,7 +68,7 @@

  
   org.apache.hadoop
-  hadoop-client
+  hadoop-common
   ${version.hadoop}
 
 

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/eb573a51/query-war/src/main/webapp/WEB-INF/web.xml
--
diff --git a/query-war/src/main/webapp/WEB-INF/web.xml 
b/query-war/src/main/webapp/WEB-INF/web.xml
index a4ff03a..ffde45a 100644
--- a/query-war/src/main/webapp/WEB-INF/web.xml
+++

[3/3] accumulo-wikisearch git commit: Fix to use proper Accumulo API

2017-02-17 Thread mmiller 
Fix to use proper Accumulo API


Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/e51ad6a2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/e51ad6a2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/e51ad6a2

Branch: refs/heads/master
Commit: e51ad6a2f6a2f6437f627e6ec7d76f17d0378323
Parents: fb4af6c
Author: Mike Miller 
Authored: Fri Feb 17 12:27:58 2017 -0500
Committer: Mike Miller 
Committed: Fri Feb 17 12:27:58 2017 -0500

--
 .../iterator/DefaultIteratorEnvironment.java| 45 
 1 file changed, 37 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/e51ad6a2/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
--
diff --git 
a/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
 
b/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
index d37751e..6783efe 100644
--- 
a/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
+++ 
b/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
@@ -16,26 +16,25 @@
  */
 package org.apache.accumulo.examples.wikisearch.iterator;
 
-import java.io.IOException;
-
-import org.apache.accumulo.core.client.impl.BaseIteratorEnvironment;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.client.sample.SamplerConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.system.MapFileIterator;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+
+import java.io.IOException;
 
-public class DefaultIteratorEnvironment extends BaseIteratorEnvironment {
+public class DefaultIteratorEnvironment implements IteratorEnvironment {
 
   AccumuloConfiguration conf;
 
-  public DefaultIteratorEnvironment(AccumuloConfiguration conf) {
-this.conf = conf;
-  }
-
   public DefaultIteratorEnvironment() {
 this.conf = AccumuloConfiguration.getDefaultConfiguration();
   }
@@ -56,4 +55,34 @@ public class DefaultIteratorEnvironment extends 
BaseIteratorEnvironment {
   public boolean isSamplingEnabled() {
 return false;
   }
+
+  @Override
+  public IteratorScope getIteratorScope() {
+throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean isFullMajorCompaction() {
+throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void registerSideChannel(SortedKeyValueIterator iter) {
+throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public Authorizations getAuthorizations() {
+throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public SamplerConfiguration getSamplerConfiguration() {
+throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public IteratorEnvironment cloneWithSamplingEnabled() {
+throw new UnsupportedOperationException();
+  }
 }