[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-09-03 Thread mmiller
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit d04109d06a41a7ae190941380ee7afb36fa2b4ef
Merge: 1c19584 d077b46
Author: Mike Miller 
AuthorDate: Tue Sep 3 11:52:56 2019 -0400

Merge branch '1.9' into 2.0

 Conflicts resolved:

server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java

 .../main/java/org/apache/accumulo/master/TabletGroupWatcher.java   | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --cc 
server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index db00210,6c98f72..b801e3b
--- 
a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@@ -102,9 -103,9 +102,8 @@@ abstract class TabletGroupWatcher exten
private static final int ASSINGMENT_BUFFER_MAX_LENGTH = 4096;
  
private final Master master;
-   final TabletStateStore store;
-   final TabletGroupWatcher dependentWatcher;
- 
+   private final TabletStateStore store;
+   private final TabletGroupWatcher dependentWatcher;
 -
final TableStats stats = new TableStats();
private SortedSet lastScanServers = 
ImmutableSortedSet.of();
  



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-29 Thread mmiller
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f4931827bcba4189181473681c6379a744256b59
Merge: 7a83445 1c902ec
Author: Mike Miller 
AuthorDate: Thu Aug 29 13:19:00 2019 -0400

Merge branch '1.9' into 2.0

 .../apache/accumulo/shell/commands/ImportDirectoryCommand.java   | 9 +
 1 file changed, 9 insertions(+)




[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-15 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 6ae7ddb99d10635e713742f35fc1db0ce9332ba2
Merge: 95c92ed ab90032
Author: Christopher Tubbs 
AuthorDate: Thu Aug 15 15:45:02 2019 -0400

Merge branch '1.9' into 2.0

 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  | 10 --
 .../org/apache/accumulo/gc/SimpleGarbageCollector.java | 15 ++-
 .../accumulo/gc/GarbageCollectWriteAheadLogsTest.java  | 18 ++
 3 files changed, 36 insertions(+), 7 deletions(-)

diff --cc 
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 289e69d,7afcdc8..e671219
--- 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -85,18 -91,26 +85,15 @@@ public class GarbageCollectWriteAheadLo
 * @param useTrash
 *  true to move files to trash rather than delete them
 */
-   GarbageCollectWriteAheadLogs(final ServerContext context, VolumeManager fs, 
boolean useTrash) {
 -  GarbageCollectWriteAheadLogs(final AccumuloServerContext context, final 
VolumeManager fs,
 -  final LiveTServerSet liveServers, boolean useTrash) throws IOException {
++  GarbageCollectWriteAheadLogs(final ServerContext context, final 
VolumeManager fs,
++  final LiveTServerSet liveServers, boolean useTrash) {
  this.context = context;
  this.fs = fs;
  this.useTrash = useTrash;
- this.liveServers = new LiveTServerSet(context, (current, deleted, added) 
-> {
-   log.debug("New tablet servers noticed: {}", added);
-   log.debug("Tablet servers removed: {}", deleted);
- });
- liveServers.startListeningForTabletServerChanges();
+ this.liveServers = liveServers;
 -
 -this.walMarker = new WalStateManager(context.getInstance(), 
ZooReaderWriter.getInstance());
 -this.store = new Iterable() {
 -  @Override
 -  public Iterator iterator() {
 -try {
 -  return Iterators.concat(new ZooTabletStateStore().iterator(),
 -  new RootTabletStateStore(context).iterator(),
 -  new MetaDataStateStore(context).iterator());
 -} catch (DistributedStoreException e) {
 -  throw new RuntimeException(e);
 -}
 -  }
 -};
 +this.walMarker = new WalStateManager(context);
 +this.store = () -> Iterators.concat(new 
ZooTabletStateStore(context).iterator(),
 +new RootTabletStateStore(context).iterator(), new 
MetaDataStateStore(context).iterator());
}
  
/**
@@@ -124,34 -138,31 +121,35 @@@
}
  
public void collect(GCStatus status) {
 -
 -Span span = Trace.start("getCandidates");
  try {
 -  status.currentLog.started = System.currentTimeMillis();
 -
 -  Map recoveryLogs = getSortedWALogs();
 -
 -  Map> logsByServer = new HashMap<>();
 -  Map> logsState = new HashMap<>();
 -  // Scan for log file info first: the order is important
 -  // Consider:
 -  // * get live servers
 -  // * new server gets a lock, creates a log
 -  // * get logs
 -  // * the log appears to belong to a dead server
 -  long count = getCurrent(logsByServer, logsState);
 -  long fileScanStop = System.currentTimeMillis();
 -
 -  log.info(String.format("Fetched %d files for %d servers in %.2f 
seconds", count,
 -  logsByServer.size(), (fileScanStop - status.currentLog.started) / 
1000.));
 -  status.currentLog.candidates = count;
 -  span.stop();
 +  long count;
 +  long fileScanStop;
 +  Map> logsByServer;
 +  Map> logsState;
 +  Map recoveryLogs;
 +  try (TraceScope span = Trace.startSpan("getCandidates")) {
 +status.currentLog.started = System.currentTimeMillis();
 +
 +recoveryLogs = getSortedWALogs();
 +
 +logsByServer = new HashMap<>();
 +logsState = new HashMap<>();
 +// Scan for log file info first: the order is important
 +// Consider:
 +// * get live servers
 +// * new server gets a lock, creates a log
 +// * get logs
 +// * the log appears to belong to a dead server
 +count = getCurrent(logsByServer, logsState);
 +fileScanStop = System.currentTimeMillis();
 +
 +log.info(String.format("Fetched %d files for %d servers in %.2f 
seconds", count,
 +logsByServer.size(), (fileScanStop - status.currentLog.started) / 
1000.));
 +status.currentLog.candidates = count;
 +  }
  
// now it's safe to get the liveServers
+   liveServers.scanServers();
Set currentServers = liveServers.getCurrentServers();
  
Map uuidToTServer;
diff --cc 
server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 147239a,24ce61c..2a0a19f
--- 

[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-15 Thread kturner
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 95c92ed95404ec1663ea97d8b4b00d58c26ba762
Merge: f63da3d 556c29e
Author: Keith Turner 
AuthorDate: Thu Aug 15 14:01:51 2019 -0400

Merge branch '1.9' into 2.0

 .../functional/TabletStateChangeIteratorIT.java| 73 --
 1 file changed, 53 insertions(+), 20 deletions(-)

diff --cc 
test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index b83e735,2fd8589..35c53cc
--- 
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++ 
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@@ -35,7 -34,10 +36,9 @@@ import org.apache.accumulo.core.client.
  import org.apache.accumulo.core.client.BatchDeleter;
  import org.apache.accumulo.core.client.BatchWriter;
  import org.apache.accumulo.core.client.BatchWriterConfig;
 -import org.apache.accumulo.core.client.Connector;
+ import org.apache.accumulo.core.client.IsolatedScanner;
  import org.apache.accumulo.core.client.MutationsRejectedException;
+ import org.apache.accumulo.core.client.RowIterator;
  import org.apache.accumulo.core.client.Scanner;
  import org.apache.accumulo.core.client.TableExistsException;
  import org.apache.accumulo.core.client.TableNotFoundException;
@@@ -81,70 -81,70 +84,75 @@@ public class TabletStateChangeIteratorI
@Test
public void test() throws AccumuloException, AccumuloSecurityException, 
TableExistsException,
TableNotFoundException {
 -String[] tables = getUniqueNames(6);
 -final String t1 = tables[0];
 -final String t2 = tables[1];
 -final String t3 = tables[2];
 -final String metaCopy1 = tables[3];
 -final String metaCopy2 = tables[4];
 -final String metaCopy3 = tables[5];
 -
 -// create some metadata
 -createTable(t1, true);
 -createTable(t2, false);
 -createTable(t3, true);
 -
 -// examine a clone of the metadata table, so we can manipulate it
 -copyTable(MetadataTable.NAME, metaCopy1);
 -
 -State state = new State();
 -while (findTabletsNeedingAttention(metaCopy1, state) > 0) {
 -  UtilWaitThread.sleep(500);
 -  copyTable(MetadataTable.NAME, metaCopy1);
 -}
 -assertEquals("No tables should need attention", 0,
 -findTabletsNeedingAttention(metaCopy1, state));
 -
 -// The metadata table stabilized and metaCopy1 contains a copy suitable 
for testing. Before
 -// metaCopy1 is modified, copy it for subsequent test.
 -copyTable(metaCopy1, metaCopy2);
 -copyTable(metaCopy1, metaCopy3);
 -
 -// test the assigned case (no location)
 -removeLocation(metaCopy1, t3);
 -assertEquals("Should have two tablets without a loc", 2,
 -findTabletsNeedingAttention(metaCopy1, state));
 -
 -// test the cases where the assignment is to a dead tserver
 -reassignLocation(metaCopy2, t3);
 -assertEquals("Should have one tablet that needs to be unassigned", 1,
 -findTabletsNeedingAttention(metaCopy2, state));
 -
 -// test the cases where there is ongoing merges
 -state = new State() {
 -  @Override
 -  public Collection merges() {
 -String tableIdToModify = 
getConnector().tableOperations().tableIdMap().get(t3);
 -return Collections.singletonList(
 -new MergeInfo(new KeyExtent(tableIdToModify, null, null), 
MergeInfo.Operation.MERGE));
 +
 +try (AccumuloClient client = 
Accumulo.newClient().from(getClientProps()).build()) {
 +
-   String[] tables = getUniqueNames(4);
++  String[] tables = getUniqueNames(6);
 +  final String t1 = tables[0];
 +  final String t2 = tables[1];
 +  final String t3 = tables[2];
-   final String cloned = tables[3];
++  final String metaCopy1 = tables[3];
++  final String metaCopy2 = tables[4];
++  final String metaCopy3 = tables[5];
 +
 +  // create some metadata
 +  createTable(client, t1, true);
 +  createTable(client, t2, false);
 +  createTable(client, t3, true);
 +
 +  // examine a clone of the metadata table, so we can manipulate it
-   cloneMetadataTable(client, cloned);
++  copyTable(client, MetadataTable.NAME, metaCopy1);
 +
 +  State state = new State(client);
-   while (findTabletsNeedingAttention(client, cloned, state) > 0) {
++  while (findTabletsNeedingAttention(client, metaCopy1, state) > 0) {
 +UtilWaitThread.sleep(500);
++copyTable(client, MetadataTable.NAME, metaCopy1);
}
 -};
 -assertEquals("Should have 2 tablets that need to be chopped or 
unassigned", 1,
 -findTabletsNeedingAttention(metaCopy2, state));
 -
 -// test the bad tablet location state case (inconsistent metadata)
 -state = new State();
 -addDuplicateLocation(metaCopy3, t3);
 -assertEquals("Should have 1 tablet that 

[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-15 Thread mmiller
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f63da3d4ce709402feb2169eeda4c9c290ad5b7b
Merge: c573676 e770862
Author: Mike Miller 
AuthorDate: Thu Aug 15 10:08:31 2019 -0400

Merge branch '1.9' into 2.0




[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-14 Thread kturner
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit c5736768d8ff3cb8799b5e254d7df36f1850a648
Merge: 5c58b08 54b2c32
Author: Keith Turner 
AuthorDate: Wed Aug 14 16:55:06 2019 -0400

Merge branch '1.9' into 2.0

 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  |  5 +++--
 .../apache/accumulo/master/FateServiceHandler.java |  3 ++-
 .../accumulo/master/util/TableValidators.java  | 22 ++
 .../accumulo/test/functional/CloneTestIT.java  | 16 
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --cc 
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 5bb8201,23c4932..289e69d
--- 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -53,6 -60,8 +53,7 @@@ import org.apache.accumulo.server.maste
  import org.apache.accumulo.server.master.state.TServerInstance;
  import org.apache.accumulo.server.master.state.TabletLocationState;
  import org.apache.accumulo.server.master.state.TabletState;
+ import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 -import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.Text;
@@@ -88,14 -97,28 +89,14 @@@ public class GarbageCollectWriteAheadLo
  this.context = context;
  this.fs = fs;
  this.useTrash = useTrash;
 -this.liveServers = new LiveTServerSet(context, new Listener() {
 -  @Override
 -  public void update(LiveTServerSet current, Set deleted,
 -  Set added) {
 -log.debug("New tablet servers noticed: " + added);
 -log.debug("Tablet servers removed: " + deleted);
 -  }
 +this.liveServers = new LiveTServerSet(context, (current, deleted, added) 
-> {
 +  log.debug("New tablet servers noticed: {}", added);
 +  log.debug("Tablet servers removed: {}", deleted);
  });
  liveServers.startListeningForTabletServerChanges();
 -this.walMarker = new WalStateManager(context.getInstance(), 
ZooReaderWriter.getInstance());
 -this.store = new Iterable() {
 -  @Override
 -  public Iterator iterator() {
 -try {
 -  return Iterators.concat(new ZooTabletStateStore().iterator(),
 -  new RootTabletStateStore(context).iterator(),
 -  new MetaDataStateStore(context).iterator());
 -} catch (DistributedStoreException e) {
 -  throw new RuntimeException(e);
 -}
 -  }
 -};
 +this.walMarker = new WalStateManager(context);
- this.store = () -> Iterators.concat(new 
RootTabletStateStore(context).iterator(),
- new MetaDataStateStore(context).iterator());
++this.store = () -> Iterators.concat(new 
ZooTabletStateStore(context).iterator(),
++new RootTabletStateStore(context).iterator(), new 
MetaDataStateStore(context).iterator());
}
  
/**
diff --cc 
server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
index c19ee7b,7949ac2..0cf7c14
--- 
a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
@@@ -250,12 -225,12 +251,12 @@@ class FateServiceHandler implements Fat
case TABLE_CLONE: {
  TableOperation tableOp = TableOperation.CLONE;
  validateArgumentCount(arguments, tableOp, 2);
- TableId srcTableId = validateTableIdArgument(arguments.get(0), 
tableOp, NOT_ROOT_ID);
 -String srcTableId = validateTableIdArgument(arguments.get(0), 
tableOp, CAN_CLONE);
++TableId srcTableId = validateTableIdArgument(arguments.get(0), 
tableOp, CAN_CLONE);
  String tableName = validateTableNameArgument(arguments.get(1), 
tableOp, NOT_SYSTEM);
 -String namespaceId;
 +NamespaceId namespaceId;
  try {
namespaceId =
 -  Namespaces.getNamespaceId(master.getInstance(), 
Tables.qualify(tableName).getFirst());
 +  Namespaces.getNamespaceId(master.getContext(), 
Tables.qualify(tableName).getFirst());
  } catch (NamespaceNotFoundException e) {
// shouldn't happen, but possible once cloning between namespaces 
is supported
throw new ThriftTableOperationException(null, tableName, tableOp,
diff --cc 
server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
index ddbce98,a9d3edb..4d44765
--- 
a/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
@@@ -80,6 -79,28 +80,28 @@@ public class TableValidators 
  }
};
  
 -  public static final Validator CAN_CLONE = new 

[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-02 Thread mmiller
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f49f6270e9c514c58bc03fe595dd283167f2ce55
Merge: aa1b886 d6c97fd
Author: Mike Miller 
AuthorDate: Fri Aug 2 15:35:02 2019 -0400

Merge branch '1.9' into 2.0

 core/src/main/java/org/apache/accumulo/core/conf/Experimental.java | 1 +
 1 file changed, 1 insertion(+)



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-07-29 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit fc0f4723fe9502ff7b49b7d27c96abe398e66d71
Merge: 5a6481f 1704d79
Author: Christopher Tubbs 
AuthorDate: Mon Jul 29 20:05:34 2019 -0400

Merge branch '1.9' into 2.0

 pom.xml  | 51 +--
 test/pom.xml |  5 -
 2 files changed, 5 insertions(+), 51 deletions(-)

diff --cc pom.xml
index bc9d69b,bc548a8..0988bba
--- a/pom.xml
+++ b/pom.xml
@@@ -122,18 -127,16 +122,19 @@@
  
  
  1
- 3.1.1
 -2.6.5
++3.1.2
 +2.5.0
  4.1.0-incubating
 -3.1.0-incubating
 +3.2.0-incubating
  false
 -
 -9.2.26.v20180806
 -7
 -1.7
 -1.7
 -3.0.5
 +2.9.9
 +3.0.1-b06
 +2.3.0.1
 +2.28
 +9.4.19.v20190610
++8
 +1.8
 +1.8
  
  
true
  2.0.2
@@@ -1121,24 -841,22 +,6 @@@

  

--  
- enforce-java-signatures
 -enforce-hadoop-profile
--
--  enforce
--
- process-test-classes
--
--  
- 
-   
- org.codehaus.mojo.signature
- java18
- 1.0
-   
- 
 -
 -  hadoop.profile
 -  (2|3)
 -  You should specify the Hadoop profile by 
major Hadoop generation, i.e. 2 or 3, not by a version number.
 -Use hadoop.version to use a particular Hadoop version within that 
generation.
 -
--  
--
--  
  


@@@ -1579,68 -1361,44 +1547,59 @@@
  
src/main/spotbugs/exclude-filter.xml

  
 -
  
 -  hadoop-default
 -  
 -
 -  !hadoop.profile
 -
 -  
 -  
 -2
 -2.6.4
 -  
 -
 -
 -
 -  hadoop2
 -  
 -
 -  hadoop.profile
 -  2
 -
 -  
 -  
 -2.6.4
 -  
 +  
 +  use-errorprone
 +  
 +
 +  
 +org.apache.maven.plugins
 +maven-compiler-plugin
 +
 +  javac-with-errorprone
 +  true
 +  
 +-XepDisableAllChecks
 +-Xep:MissingOverride:ERROR
 +
-XepExcludedPaths:.*/(proto|thrift|generated-sources)/.*
 +  
 +
 +
 +  
 +com.google.errorprone
 +error_prone_core
 +2.3.1
 +  
 +  
 +org.codehaus.plexus
 +plexus-compiler-javac-errorprone
 +2.8
 +  
 +
 +  
 +
 +  
  
 -
  
 -  hadoop3
 -  
 -
 -  hadoop.profile
 -  3
 -
 -  
 -  
 -3.0.0
 -  
 +  sec-bugs
 +  
 +
 +  
 +com.github.spotbugs
 +spotbugs-maven-plugin
 +
 +  
 +
 +  com.h3xstream.findsecbugs
 +  findsecbugs-plugin
 +  1.8.0
 +
 +  
 +
 +  
 +
 +  
  
- 
-   jdk-release-flag
-   
- [9,)
-   
-   
- 8
-   
- 

  



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-07-29 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 5a6481f7c15253bfc19a0ded5a96cb02ce9edb7e
Merge: 0bb3f89 fd4d478
Author: Christopher Tubbs 
AuthorDate: Mon Jul 29 15:08:16 2019 -0400

Merge branch '1.9' into 2.0

 pom.xml| 4 
 test/src/main/java/org/apache/accumulo/test/ShellServerIT.java | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --cc pom.xml
index c08d56c,ff3f20b..bc9d69b
--- a/pom.xml
+++ b/pom.xml
@@@ -1353,6 -1100,10 +1353,10 @@@
m2e.version
  

+   
 -
 -7
++
++8
+   

  




[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-07-26 Thread kturner
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 0bb3f896a28ec315976fb4570ced8e3913a75587
Merge: 129e8ec 6546fc3
Author: Keith Turner 
AuthorDate: Fri Jul 26 15:20:31 2019 -0400

Merge branch '1.9' into 2.0

 .../apache/accumulo/tserver/BusiestTracker.java|  8 --
 .../accumulo/tserver/BusiestTrackerTest.java   | 32 --
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java
index 1eaa1d6,229ed74..cf19b5a
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java
@@@ -60,9 -60,16 +60,13 @@@ public abstract class BusiestTracker 
  
counts.put(extent, count);
  
-   long delta = count - lastCounts.getOrDefault(extent, 0L);
 -  Long lastCount = lastCounts.get(extent);
++  long lastCount = lastCounts.getOrDefault(extent, 0L);
  
-   // handle case where tablet leaves tserver and returns OR tablet had no 
activity
 -  // if a tablet leaves a tserver and come back, then its count will be 
reset. This could make
++  // if a tablet leaves a tserver and comes back, then its count will be 
reset. This could make
+   // lastCount higher than the current count. That is why lastCount > 
count is checked below.
 -  if (lastCount == null || lastCount > count)
 -lastCount = 0L;
 -
 -  long delta = count - lastCount;
++  long delta = (lastCount > count) ? count : count - lastCount;
+ 
+   // handle case where tablet had no activity
if (delta > 0)
  tabletsWithDelta.add(new ComparablePair<>(delta, extent));
  }



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-07-25 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 129e8ec4abf5f94d7163dcf761df3e6687a49f59
Merge: 3a4a4d6 87da062
Author: Christopher Tubbs 
AuthorDate: Thu Jul 25 17:52:38 2019 -0400

Merge branch '1.9' into 2.0

 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --cc pom.xml
index 97cc195,78dbdd3..c08d56c
--- a/pom.xml
+++ b/pom.xml
@@@ -785,7 -567,7 +785,7 @@@
  
com.github.spotbugs
spotbugs-maven-plugin
-   ${spotbugs.version}
 -  3.1.12.1
++  ${spotbugs.version}.1

  true
  Max



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-07-25 Thread kturner
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 40d954f22b22ff437b938c6bf3607298b30e1088
Merge: d6dfee4 83429b6
Author: Keith Turner 
AuthorDate: Thu Jul 25 14:36:59 2019 -0400

Merge branch '1.9' into 2.0

 .../org/apache/accumulo/core/conf/Property.java|   5 +
 .../apache/accumulo/tserver/BusiestTracker.java|  92 +++
 .../org/apache/accumulo/tserver/TabletServer.java  |  35 +
 .../org/apache/accumulo/tserver/tablet/Tablet.java |   4 +
 .../accumulo/tserver/BusiestTrackerTest.java   | 168 +
 5 files changed, 304 insertions(+)

diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 7879ae4,494f6b2..302ce06
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -444,8 -483,13 +444,13 @@@ public enum Property 
"The minimum number of threads to use to handle incoming requests."),
TSERV_THREADCHECK("tserver.server.threadcheck.time", "1s", 
PropertyType.TIMEDURATION,
"The time between adjustments of the server thread pool."),
 -  TSERV_MAX_MESSAGE_SIZE("tserver.server.message.size.max", "1G", 
PropertyType.MEMORY,
 +  TSERV_MAX_MESSAGE_SIZE("tserver.server.message.size.max", "1G", 
PropertyType.BYTES,
"The maximum size of a message that can be sent to a tablet server."),
+   TSERV_LOG_BUSY_TABLETS_COUNT("tserver.log.busy.tablets.count", "0", 
PropertyType.COUNT,
+   "Number of busiest tablets to log. Logged at interval controlled by "
+   + "tserver.log.busy.tablets.interval. If <= 0, logging of busy 
tablets is disabled"),
+   TSERV_LOG_BUSY_TABLETS_INTERVAL("tserver.log.busy.tablets.interval", "1h",
+   PropertyType.TIMEDURATION, "Time interval between logging out busy 
tablets information."),
TSERV_HOLD_TIME_SUICIDE("tserver.hold.time.max", "5m", 
PropertyType.TIMEDURATION,
"The maximum time for a tablet server to be in the \"memory full\" 
state."
+ " If the tablet server cannot write out memory in this much time, 
it will"
diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java
index 000,ea60cad..1eaa1d6
mode 00,100644..100644
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java
@@@ -1,0 -1,96 +1,92 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.accumulo.tserver;
+ 
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ 
 -import org.apache.accumulo.core.data.impl.KeyExtent;
++import org.apache.accumulo.core.dataImpl.KeyExtent;
+ import org.apache.accumulo.core.util.ComparablePair;
+ import org.apache.accumulo.tserver.tablet.Tablet;
+ 
+ import com.google.common.collect.Ordering;
+ 
+ /**
+  * Computes the N tablets that have the highest deltas for a given 
monotonically increasing counter.
+  */
+ public abstract class BusiestTracker {
+ 
+   private Map lastCounts = Collections.emptyMap();
+   private final int numBusiestTabletsToLog;
+ 
+   BusiestTracker(int numBusiestTabletsToLog) {
+ this.numBusiestTabletsToLog = numBusiestTabletsToLog;
+   }
+ 
+   protected abstract long extractCount(Tablet tablet);
+ 
+   public List> 
computeBusiest(Collection tablets) {
+ 
+ HashMap counts = new HashMap<>();
+ 
+ ArrayList> tabletsWithDelta = new 
ArrayList<>();
+ 
+ for (Tablet tablet : tablets) {
+   KeyExtent extent = tablet.getExtent();
+ 
+   // only get the count once to ensure consistency in the case of 
multiple threads
+   long count = extractCount(tablet);
+ 
+   if (count == 0)
+ continue;
+ 
+   counts.put(extent, count);
+ 
 -  Long lastCount = lastCounts.get(extent);
 -  if (lastCount == null)
 -lastCount = 0L;
 -
 -  long delta = count - lastCount;
++  long delta = count - lastCounts.getOrDefault(extent, 0L);
+ 
+

[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-07-16 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 4724f12c390cb6c0e3a0d44e5f830d3cd029fdc8
Merge: b32d81d 6b1ff04
Author: Christopher Tubbs 
AuthorDate: Tue Jul 16 17:18:47 2019 -0400

Merge branch '1.9' into 2.0

 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --cc .travis.yml
index c883b19,b67e92a..b41a9d2
--- a/.travis.yml
+++ b/.travis.yml
@@@ -27,7 -27,7 +27,8 @@@ before_script
- unset _JAVA_OPTIONS
  env:
- BUILD_CMD="mvn clean verify javadoc:jar -DskipITs"  # main build of unit 
tests and javadoc
 -  - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs -Dhadoop.profile=3"
 +  - BUILD_CMD="travis_wait 30 mvn clean verify -PskipQA,sec-bugs 
-Dspotbugs.skip=false -Dspotbugs.timeout=180"  # check for spotbugs+security
 +  - BUILD_CMD="mvn clean package -DskipTests -Dhadoop.version=3.0.3"  # quick 
compile to verify older API
+ # suppress download logs which cause travis to exceed the 4MB build log 
output limit
  script:
-   - $BUILD_CMD
+   - $BUILD_CMD -B 
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-06-21 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit e0e9c471fc5beaba71b1b826675ab867ba47418b
Merge: 4160067 d7fccfd
Author: Christopher Tubbs 
AuthorDate: Fri Jun 21 04:20:36 2019 -0400

Merge branch '1.9' into 2.0

 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --cc .travis.yml
index c96e059,1b72b5b..c883b19
--- a/.travis.yml
+++ b/.travis.yml
@@@ -22,12 -22,11 +22,12 @@@ cache
  - $HOME/.m2
  install: echo NOOP Skipping pre-fetch of Maven dependencies
  jdk:
--  - openjdk8
++  - openjdk11
  before_script:
- unset _JAVA_OPTIONS
  env:
- BUILD_CMD="mvn clean verify javadoc:jar -DskipITs"  # main build of unit 
tests and javadoc
 -  - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs -Dhadoop.profile=3"
 +  - BUILD_CMD="travis_wait 30 mvn clean verify -PskipQA,sec-bugs 
-Dspotbugs.skip=false -Dspotbugs.timeout=180"  # check for spotbugs+security
-   - BUILD_CMD="mvn clean compile -Dhadoop.version=3.0.3"  # quick compile to 
verify older API
++  - BUILD_CMD="mvn clean package -DskipTests -Dhadoop.version=3.0.3"  # quick 
compile to verify older API
  script:
- $BUILD_CMD



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-06-20 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit ad64724fd72c26c22dad232eddd7de2f8ca96df8
Merge: 9e32263 89829d7
Author: Christopher Tubbs 
AuthorDate: Thu Jun 20 16:18:55 2019 -0400

Merge branch '1.9' into 2.0

 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)




[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-06-09 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit c95609c513a41f54a145e9b2cc06ce4435dd7dbd
Merge: 68f8f5e 0884bfd
Author: Christopher Tubbs 
AuthorDate: Sun Jun 9 22:08:19 2019 -0400

Merge branch '1.9' into 2.0

 assemble/pom.xml  | 2 +-
 core/pom.xml  | 2 +-
 hadoop-mapreduce/pom.xml  | 2 +-
 iterator-test-harness/pom.xml | 2 +-
 minicluster/pom.xml   | 2 +-
 pom.xml   | 2 +-
 server/base/pom.xml   | 2 +-
 server/gc/pom.xml | 2 +-
 server/master/pom.xml | 2 +-
 server/monitor/pom.xml| 2 +-
 server/native/pom.xml | 2 +-
 server/tracer/pom.xml | 2 +-
 server/tserver/pom.xml| 2 +-
 start/pom.xml | 2 +-
 test/pom.xml  | 2 +-
 15 files changed, 15 insertions(+), 15 deletions(-)

diff --cc hadoop-mapreduce/pom.xml
index 36c8bd3,000..f940761
mode 100644,00..100644
--- a/hadoop-mapreduce/pom.xml
+++ b/hadoop-mapreduce/pom.xml
@@@ -1,120 -1,0 +1,120 @@@
 +
 +
- http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
https://maven.apache.org/maven-v4_0_0.xsd;>
++http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
https://maven.apache.org/xsd/maven-4.0.0.xsd;>
 +  4.0.0
 +  
 +org.apache.accumulo
 +accumulo-project
 +2.0.0-SNAPSHOT
 +  
 +  accumulo-hadoop-mapreduce
 +  Apache Accumulo Hadoop MapReduce
 +  Apache Accumulo MapReduce bindings.
 +  
 +
 +  com.beust
 +  jcommander
 +
 +
 +  com.google.guava
 +  guava
 +
 +
 +  org.apache.accumulo
 +  accumulo-core
 +
 +
 +  org.apache.hadoop
 +  hadoop-client-api
 +
 +
 +  org.slf4j
 +  slf4j-api
 +
 +
 +  org.apache.hadoop
 +  hadoop-client-runtime
 +  runtime
 +
 +
 +  junit
 +  junit
 +  test
 +
 +
 +  org.apache.accumulo
 +  accumulo-minicluster
 +  test
 +
 +
 +  org.apache.accumulo
 +  accumulo-server-base
 +  test
 +
 +
 +  org.apache.accumulo
 +  accumulo-test
 +  test
 +
 +
 +  org.slf4j
 +  slf4j-log4j12
 +  test
 +
 +  
 +  
 +
 +  
 +net.revelc.code
 +apilyzer-maven-plugin
 +
 +  
 +apilyzer
 +
 +  analyze
 +
 +
 +  
 +
org[.]apache[.]accumulo[.]core[.]client[.]mapred(?:uce)?[.].*
 +  
 +  
 +.*[.]impl[.].*
 +  
 +  
 +
org[.]apache[.]accumulo[.]core[.](?:client|data|security)[.](?!.*(impl|thrift|crypto).*).*
 +
 +org[.]apache[.]accumulo[.]core[.]util[.]Pair
 +
 +org[.]apache[.]hadoop[.]conf[.]Configuration
 +org[.]apache[.]hadoop[.]fs[.](FileSystem|Path)
 +
org[.]apache[.]hadoop[.]io[.](Text|Writable|WritableComparable|WritableComparator)
 +
org[.]apache[.]hadoop[.]mapred[.](JobConf|RecordReader|InputSplit|RecordWriter|Reporter)
 +
org[.]apache[.]hadoop[.]mapred[.]FileOutputFormat[$]Counter
 +
org[.]apache[.]hadoop[.]mapreduce[.](Job|JobContext|RecordReader|InputSplit|TaskAttemptContext|RecordWriter|OutputCommitter|TaskInputOutputContext)
 +
org[.]apache[.]hadoop[.]mapreduce[.]lib[.]output[.]FileOutputFormat[$]Counter
 +org[.]apache[.]hadoop[.]util[.]Progressable
 +
org[.]apache[.]hadoop[.]mapred[.](FileAlreadyExistsException|InvalidJobConfException)
 +  
 +
 +  
 +
 +  
 +
 +  
 +



[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-06-07 Thread ctubbsii
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 3bdb6cd94d0fbdd88bff6da4aeb682bd4391c847
Merge: c8776a9 3311218
Author: Christopher Tubbs 
AuthorDate: Fri Jun 7 12:41:25 2019 -0400

Merge branch '1.9' into 2.0

 core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java | 4 +---
 core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooUtil.java   | 4 ++--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --cc core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java
index 98e49d3,000..0bddb07
mode 100644,00..100644
--- a/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java
+++ b/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java
@@@ -1,263 -1,0 +1,261 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + * http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.fate.zookeeper;
 +
 +import java.util.List;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.atomic.AtomicInteger;
 +
 +import org.apache.accumulo.fate.util.Retry;
 +import org.apache.accumulo.fate.util.Retry.RetryFactory;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.ZooKeeperConnectionInfo;
 +import org.apache.zookeeper.KeeperException;
 +import org.apache.zookeeper.KeeperException.Code;
 +import org.apache.zookeeper.Watcher;
 +import org.apache.zookeeper.ZooKeeper;
 +import org.apache.zookeeper.data.ACL;
 +import org.apache.zookeeper.data.Stat;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +public class ZooReader implements IZooReader {
 +  private static final Logger log = LoggerFactory.getLogger(ZooReader.class);
 +
 +  protected String keepers;
 +  protected int timeout;
 +  private final RetryFactory retryFactory;
-   private final ZooKeeperConnectionInfo info;
 +
 +  protected ZooKeeper getSession(String keepers, int timeout, String scheme, 
byte[] auth) {
 +return ZooSession.getSession(keepers, timeout, scheme, auth);
 +  }
 +
 +  protected ZooKeeper getZooKeeper() {
 +return getSession(keepers, timeout, null, null);
 +  }
 +
 +  protected RetryFactory getRetryFactory() {
 +return retryFactory;
 +  }
 +
 +  protected void retryOrThrow(Retry retry, KeeperException e) throws 
KeeperException {
 +log.warn("Saw (possibly) transient exception communicating with 
ZooKeeper", e);
 +if (retry.canRetry()) {
 +  retry.useRetry();
 +  return;
 +}
 +
 +log.error("Retry attempts ({}) exceeded trying to communicate with 
ZooKeeper",
 +retry.retriesCompleted());
 +throw e;
 +  }
 +
 +  @Override
 +  public byte[] getData(String zPath, Stat stat) throws KeeperException, 
InterruptedException {
 +return getData(zPath, false, stat);
 +  }
 +
 +  @Override
 +  public byte[] getData(String zPath, boolean watch, Stat stat)
 +  throws KeeperException, InterruptedException {
 +final Retry retry = getRetryFactory().createRetry();
 +while (true) {
 +  try {
 +return getZooKeeper().getData(zPath, watch, stat);
 +  } catch (KeeperException e) {
 +final Code code = e.code();
 +if (code == Code.CONNECTIONLOSS || code == Code.OPERATIONTIMEOUT
 +|| code == Code.SESSIONEXPIRED) {
 +  retryOrThrow(retry, e);
 +} else {
 +  throw e;
 +}
 +  }
 +
 +  retry.waitForNextAttempt();
 +}
 +  }
 +
 +  @Override
 +  public byte[] getData(String zPath, Watcher watcher, Stat stat)
 +  throws KeeperException, InterruptedException {
 +final Retry retry = getRetryFactory().createRetry();
 +while (true) {
 +  try {
 +return getZooKeeper().getData(zPath, watcher, stat);
 +  } catch (KeeperException e) {
 +final Code code = e.code();
 +if (code == Code.CONNECTIONLOSS || code == Code.OPERATIONTIMEOUT
 +|| code == Code.SESSIONEXPIRED) {
 +  retryOrThrow(retry, e);
 +} else {
 +  throw e;
 +}
 +  }
 +
 +  retry.waitForNextAttempt();
 +}
 +  }
 +
 +  @Override
 +  public Stat getStatus(String zPath) throws KeeperException, 
InterruptedException {
 +

[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-05-28 Thread kturner
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 21c6a0e489225f3206be19c92fe0c1904cb7825c
Merge: 3b010ca 057f14e
Author: Keith Turner 
AuthorDate: Tue May 28 15:17:51 2019 -0400

Merge branch '1.9' into 2.0