date:20170719

[2/2] accumulo git commit: Merge branch '1.8'

2017-07-19 Thread ibella

Merge branch '1.8'


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/80762e9f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/80762e9f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/80762e9f

Branch: refs/heads/master
Commit: 80762e9fcb25c1772510119d466c3352cec32f82
Parents: 32dbd42 6c20e50
Author: Ivan Bella 
Authored: Wed Jul 19 18:51:17 2017 -0400
Committer: Ivan Bella 
Committed: Wed Jul 19 18:51:17 2017 -0400

--
 .../balancer/HostRegexTableLoadBalancer.java| 117 +--
 .../BaseHostRegexTableLoadBalancerTest.java |  65 ++-
 ...gexTableLoadBalancerReconfigurationTest.java |   2 +-
 .../HostRegexTableLoadBalancerTest.java |  61 --
 4 files changed, 224 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/80762e9f/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
--
diff --cc 
server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 945efd0,dbf03d0..c3d15c8
--- 
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@@ -31,16 -32,19 +32,20 @@@ import java.util.SortedMap
  import java.util.TreeMap;
  import java.util.regex.Pattern;
  
+ import com.google.common.collect.HashMultimap;
+ import com.google.common.collect.Iterables;
+ import com.google.common.collect.Multimap;
  import org.apache.accumulo.core.client.admin.TableOperations;
 -import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.client.impl.Table;
  import org.apache.accumulo.core.conf.ConfigurationObserver;
 +import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
  import org.apache.accumulo.core.conf.Property;
  import org.apache.accumulo.core.data.impl.KeyExtent;
+ import org.apache.accumulo.core.master.thrift.TableInfo;
  import org.apache.accumulo.core.master.thrift.TabletServerStatus;
  import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 +import org.apache.accumulo.server.AccumuloServerContext;
  import org.apache.accumulo.server.conf.ServerConfiguration;
 -import org.apache.accumulo.server.conf.ServerConfigurationFactory;
  import org.apache.accumulo.server.master.state.TServerInstance;
  import org.apache.accumulo.server.master.state.TabletMigration;
  import org.apache.commons.lang.builder.ToStringBuilder;
@@@ -77,15 -88,24 +84,24 @@@ public class HostRegexTableLoadBalance
+ "balancer.host.regex.concurrent.migrations";
private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
+   private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0;
+   public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY = 
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+   + "balancer.host.regex.max.outstanding.migrations";
  
 -  protected long oobCheckMillis = 
AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
 +  protected long oobCheckMillis = 
ConfigurationTypeHelper.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
  
+   private static final long ONE_HOUR = 60 * 60 * 1000;
 -  private static final Set EMPTY_MIGRATIONS = 
Collections.EMPTY_SET;
++  private static final Set EMPTY_MIGRATIONS = 
Collections.emptySet();
+ 
 -  private Map tableIdToTableName = null;
 +  private Map tableIdToTableName = null;
private Map poolNameToRegexPattern = null;
private volatile long lastOOBCheck = System.currentTimeMillis();
-   private boolean isIpBasedRegex = false;
+   private volatile boolean isIpBasedRegex = false;
private Map> pools = 
new HashMap<>();
-   private int maxTServerMigrations = 
HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
+   private volatile int maxTServerMigrations = 
HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
+   private volatile int maxOutstandingMigrations = 
DEFAULT_OUTSTANDING_MIGRATIONS;
 -  private final Map migrationsFromLastPass = new 
HashMap();
 -  private final Map tableToTimeSinceNoMigrations = new 
HashMap();
++  private final Map migrationsFromLastPass = new 
HashMap<>();
++  private final Map tableToTimeSinceNoMigrations = new 
HashMap<>();
  
/**
 * Group the set of current tservers by pool name. Tservers that don't 
match a regex are

[1/2] accumulo git commit: ACCUMULO-4654 Continue balancing even with pending migrations

2017-07-19 Thread ibella

Repository: accumulo
Updated Branches:
  refs/heads/master 32dbd4258 -> 80762e9fc


ACCUMULO-4654 Continue balancing even with pending migrations

Added the ability to continue balancing with the HostRegexTableLoadBalancer 
even if there are pending migrations.
Also added detection for tables that have been continuously migrating for over 
an hour.

Closes #272


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c20e50c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c20e50c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c20e50c

Branch: refs/heads/master
Commit: 6c20e50c223a44deaccf4e0690ccec9b5749b21c
Parents: cdafd02
Author: Ivan Bella 
Authored: Mon Jun 19 13:57:44 2017 -0400
Committer: Ivan Bella 
Committed: Wed Jul 19 17:20:24 2017 -0400

--
 .../balancer/HostRegexTableLoadBalancer.java| 117 +--
 .../BaseHostRegexTableLoadBalancerTest.java |  65 ++-
 ...gexTableLoadBalancerReconfigurationTest.java |   2 +-
 .../HostRegexTableLoadBalancerTest.java |  59 +-
 4 files changed, 224 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
--
diff --git 
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
 
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 5207230..dbf03d0 100644
--- 
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.master.balancer;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -31,11 +32,15 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
 import org.apache.accumulo.core.client.admin.TableOperations;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.ConfigurationObserver;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -65,7 +70,9 @@ import org.slf4j.LoggerFactory;
  * table.custom.balancer.host.regex.is.ip=true
  * It's possible that this balancer may create a lot of migrations. To limit 
the number of migrations that are created during a balance call, set the 
following
  * property (default 250):
- * table.custom.balancer.host.regex.concurrent.migrations
+ * table.custom.balancer.host.regex.concurrent.migrations This balancer 
can continue balancing even if there are outstanding migrations. To limit the
+ * number of outstanding migrations in which this balancer will continue 
balancing, set the following property (default 0):
+ * table.custom.balancer.host.regex.max.outstanding.migrations
  *
  */
 public class HostRegexTableLoadBalancer extends TableLoadBalancer implements 
ConfigurationObserver {
@@ -81,15 +88,24 @@ public class HostRegexTableLoadBalancer extends 
TableLoadBalancer implements Con
   + "balancer.host.regex.concurrent.migrations";
   private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
   protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
+  private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0;
+  public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY = 
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+  + "balancer.host.regex.max.outstanding.migrations";
 
   protected long oobCheckMillis = 
AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
 
+  private static final long ONE_HOUR = 60 * 60 * 1000;
+  private static final Set EMPTY_MIGRATIONS = Collections.EMPTY_SET;
+
   private Map tableIdToTableName = null;
   private Map poolNameToRegexPattern = null;
   private volatile long lastOOBCheck = System.currentTimeMillis();
-  private boolean isIpBasedRegex = false;
+  private volatile boolean isIpBasedRegex = false;

[2/2] accumulo git commit: Merge branch '1.7' into 1.8

2017-07-19 Thread mmiller

Merge branch '1.7' into 1.8


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a5ed1ba3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a5ed1ba3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a5ed1ba3

Branch: refs/heads/1.8
Commit: a5ed1ba3a90652d432d7ca3db927f06a5052e036
Parents: 6c20e50 818e56b
Author: Mike Miller 
Authored: Wed Jul 19 18:17:39 2017 -0400
Committer: Mike Miller 
Committed: Wed Jul 19 18:17:39 2017 -0400

--
 .../accumulo/test/AssignmentThreadsIT.java  | 103 ---
 .../accumulo/test/VerifySerialRecoveryIT.java   |  20 +++-
 2 files changed, 18 insertions(+), 105 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/a5ed1ba3/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
--
diff --cc 
test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
index 3ec0e30,000..1f95e08
mode 100644,00..100644
--- a/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
@@@ -1,107 -1,0 +1,123 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + * http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertFalse;
 +import static org.junit.Assert.assertTrue;
 +
++import java.util.Random;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.minicluster.ServerType;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.minicluster.impl.ProcessReference;
 +import org.apache.accumulo.server.util.Admin;
 +import org.apache.accumulo.test.functional.ConfigurableMacBase;
 +import org.apache.accumulo.test.functional.FunctionalTestUtils;
 +import org.apache.accumulo.tserver.TabletServer;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.RawLocalFileSystem;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +import com.google.common.collect.Iterators;
 +
 +public class VerifySerialRecoveryIT extends ConfigurableMacBase {
 +
++  private final static byte[] HEXCHARS = {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 
0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66};
++  private final static Random random = new Random();
++
++  public static byte[] randomHex(int n) {
++byte[] binary = new byte[n];
++byte[] hex = new byte[n * 2];
++random.nextBytes(binary);
++int count = 0;
++for (byte x : binary) {
++  hex[count++] = HEXCHARS[(x >> 4) & 0xf];
++  hex[count++] = HEXCHARS[x & 0xf];
++}
++return hex;
++  }
++
 +  @Override
 +  public void configure(MiniAccumuloConfigImpl cfg, Configuration 
hadoopCoreSite) {
 +cfg.setNumTservers(1);
 +cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
 +cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "20");
 +hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
 +  }
 +
 +  @Test(timeout = 4 * 60 * 1000)
 +  public void testSerializedRecovery() throws Exception {
 +// make a table with many splits
 +String tableName = getUniqueNames(1)[0];
 +Connector c = getConnector();
 +c.tableOperations().create(tableName);
 +SortedSet splits = new TreeSet<>();
 +for (int i = 0; i < 200; i++) {
-   splits.add(new Text(AssignmentThreadsIT.randomHex(8)));
++  splits.add(new Text(randomHex(8)));
 +}
 +c.tableOperations().addSplits(tableName, splits);
 +// load data to give the recovery something to do
 +BatchWriter bw = c.createBatchWriter(tableName, null);
 +

accumulo git commit: ACCUMULO-4602 Deleted AssignmentThreadsIT

2017-07-19 Thread mmiller

Repository: accumulo
Updated Branches:
  refs/heads/1.7 b7d9be095 -> 818e56bb5


ACCUMULO-4602 Deleted AssignmentThreadsIT


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/818e56bb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/818e56bb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/818e56bb

Branch: refs/heads/1.7
Commit: 818e56bb54499079bf86ad80e5e619b248d6ff69
Parents: b7d9be0
Author: Mike Miller 
Authored: Wed Jul 19 17:16:39 2017 -0400
Committer: Mike Miller 
Committed: Wed Jul 19 18:14:41 2017 -0400

--
 .../accumulo/test/AssignmentThreadsIT.java  | 98 
 .../accumulo/test/VerifySerialRecoveryIT.java   | 20 +++-
 2 files changed, 18 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/818e56bb/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
--
diff --git 
a/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java 
b/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
deleted file mode 100644
index 3778de6..000
--- a/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.Random;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-// ACCUMULO-1177
-public class AssignmentThreadsIT extends ConfigurableMacIT {
-
-  @Override
-  public void configure(MiniAccumuloConfigImpl cfg, Configuration 
hadoopCoreSite) {
-cfg.setNumTservers(1);
-cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "1");
-  }
-
-  // [0-9a-f]
-  private final static byte[] HEXCHARS = {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 
0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66};
-  private final static Random random = new Random();
-
-  public static byte[] randomHex(int n) {
-byte[] binary = new byte[n];
-byte[] hex = new byte[n * 2];
-random.nextBytes(binary);
-int count = 0;
-for (byte x : binary) {
-  hex[count++] = HEXCHARS[(x >> 4) & 0xf];
-  hex[count++] = HEXCHARS[x & 0xf];
-}
-return hex;
-  }
-
-  @Test(timeout = 5 * 60 * 1000)
-  public void testConcurrentAssignmentPerformance() throws Exception {
-// make a table with a lot of splits
-String tableName = getUniqueNames(1)[0];
-Connector c = getConnector();
-log.info("Creating table");
-c.tableOperations().create(tableName);
-SortedSet splits = new TreeSet<>();
-for (int i = 0; i < 1000; i++) {
-  splits.add(new Text(randomHex(8)));
-}
-log.info("Adding splits");
-c.tableOperations().addSplits(tableName, splits);
-log.info("Taking table offline");
-c.tableOperations().offline(tableName, true);
-
-// time how long it takes to load
-log.info("Bringing the table online");
-long now = System.nanoTime();
-c.tableOperations().online(tableName, true);
-long diff = System.nanoTime() - now;
-
-log.info("Loaded " + splits.size() + " tablets in " + diff + " ns");
-
c.instanceOperations().setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT.getKey(),
 "20");
-log.info("Taking table offline, again");
-c.tableOperations().offline(tableName, true);
-// wait >10 seconds for thread pool to update
-log.info("sleep for {} ms so thread pool can update", 60_000);
-UtilWaitThread.sleep(60_000);
-
-log.info("Bringing table back online");
-now =

accumulo git commit: ACCUMULO-4654 Continue balancing even with pending migrations

2017-07-19 Thread ibella

Repository: accumulo
Updated Branches:
  refs/heads/1.8 cdafd0203 -> 6c20e50c2


ACCUMULO-4654 Continue balancing even with pending migrations

Added the ability to continue balancing with the HostRegexTableLoadBalancer 
even if there are pending migrations.
Also added detection for tables that have been continuously migrating for over 
an hour.

Closes #272


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c20e50c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c20e50c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c20e50c

Branch: refs/heads/1.8
Commit: 6c20e50c223a44deaccf4e0690ccec9b5749b21c
Parents: cdafd02
Author: Ivan Bella 
Authored: Mon Jun 19 13:57:44 2017 -0400
Committer: Ivan Bella 
Committed: Wed Jul 19 17:20:24 2017 -0400

--
 .../balancer/HostRegexTableLoadBalancer.java| 117 +--
 .../BaseHostRegexTableLoadBalancerTest.java |  65 ++-
 ...gexTableLoadBalancerReconfigurationTest.java |   2 +-
 .../HostRegexTableLoadBalancerTest.java |  59 +-
 4 files changed, 224 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c20e50c/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
--
diff --git 
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
 
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 5207230..dbf03d0 100644
--- 
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.master.balancer;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -31,11 +32,15 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
 import org.apache.accumulo.core.client.admin.TableOperations;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.ConfigurationObserver;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -65,7 +70,9 @@ import org.slf4j.LoggerFactory;
  * table.custom.balancer.host.regex.is.ip=true
  * It's possible that this balancer may create a lot of migrations. To limit 
the number of migrations that are created during a balance call, set the 
following
  * property (default 250):
- * table.custom.balancer.host.regex.concurrent.migrations
+ * table.custom.balancer.host.regex.concurrent.migrations This balancer 
can continue balancing even if there are outstanding migrations. To limit the
+ * number of outstanding migrations in which this balancer will continue 
balancing, set the following property (default 0):
+ * table.custom.balancer.host.regex.max.outstanding.migrations
  *
  */
 public class HostRegexTableLoadBalancer extends TableLoadBalancer implements 
ConfigurationObserver {
@@ -81,15 +88,24 @@ public class HostRegexTableLoadBalancer extends 
TableLoadBalancer implements Con
   + "balancer.host.regex.concurrent.migrations";
   private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
   protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
+  private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0;
+  public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY = 
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+  + "balancer.host.regex.max.outstanding.migrations";
 
   protected long oobCheckMillis = 
AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
 
+  private static final long ONE_HOUR = 60 * 60 * 1000;
+  private static final Set EMPTY_MIGRATIONS = Collections.EMPTY_SET;
+
   private Map tableIdToTableName = null;
   private Map poolNameToRegexPattern = null;
   private volatile long lastOOBCheck = System.currentTimeMillis();
-  private boolean isIpBasedRegex = false;
+  private volatile boolean isIpBasedRegex = false;

accumulo git commit: ACCUMULO-4061 Added code to determine Accumulo version from running tserver

2017-07-19 Thread mmiller

Repository: accumulo
Updated Branches:
  refs/heads/master d6ea49e03 -> 32dbd4258


ACCUMULO-4061 Added code to determine Accumulo version from running tserver


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/32dbd425
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/32dbd425
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/32dbd425

Branch: refs/heads/master
Commit: 32dbd4258c0f41fd84cca08b88948b853c8a2845
Parents: d6ea49e
Author: Luis Tavarez 
Authored: Tue Jul 18 19:58:58 2017 -0400
Committer: Mike Miller 
Committed: Wed Jul 19 12:12:06 2017 -0400

--
 .../core/master/thrift/TabletServerStatus.java  | 112 ++-
 core/src/main/thrift/master.thrift  |   1 +
 .../apache/accumulo/tserver/TabletServer.java   |   5 +
 3 files changed, 114 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/accumulo/blob/32dbd425/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
--
diff --git 
a/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
 
b/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
index c0756cd..3406a03 100644
--- 
a/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
@@ -41,6 +41,7 @@ public class TabletServerStatus implements 
org.apache.thrift.TBase bulkImports; // required
+  public java.lang.String version; // required
 
   /** The set of fields this struct contains, along with convenience methods 
for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -75,7 +77,8 @@ public class TabletServerStatus implements 
org.apache.thrift.TBase byName = new 
java.util.HashMap();
 
@@ -118,6 +121,8 @@ public class TabletServerStatus implements 
org.apache.thrift.TBase logSorts,
 long flushs,
 long syncs,
-java.util.List bulkImports)
+java.util.List bulkImports,
+java.lang.String version)
   {
 this();
 this.tableMap = tableMap;
@@ -252,6 +260,7 @@ public class TabletServerStatus implements 
org.apache.thrift.TBasehttp://git-wip-us.apache.org/repos/asf/accumulo/blob/32dbd425/core/src/main/thrift/master.thrift
--
diff --git a/core/src/main/thrift/master.thrift 
b/core/src/main/thrift/master.thrift
index fbacc66..8633890 100644
--- a/core/src/main/thrift/master.thrift
+++ b/core/src/main/thrift/master.thrift
@@ -85,6 +85,7 @@ struct TabletServerStatus {
   15:i64 flushs
   16:i64 syncs
   17:list bulkImports
+  18:string version
 }
 
 enum MasterState {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/32dbd425/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
--
diff --git 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 7e6188c..7584ee4 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -416,6 +416,10 @@ public class TabletServer extends AccumuloServerContext 
implements Runnable {
 }
   }
 
+  public String getVersion() {
+return Constants.VERSION;
+  }
+
   private static long jitter(long ms) {
 Random r = new Random();
 // add a random 10% wait
@@ -3057,6 +3061,7 @@ public class TabletServer extends AccumuloServerContext 
implements Runnable {
 result.bulkImports = new ArrayList<>();
 result.bulkImports.addAll(clientHandler.getBulkLoadStatus());
 result.bulkImports.addAll(bulkImportStatus.getBulkLoadStatus());
+result.version = getVersion();
 return result;
   }

[2/2] accumulo git commit: Merge branch '1.8'

[1/2] accumulo git commit: ACCUMULO-4654 Continue balancing even with pending migrations

[2/2] accumulo git commit: Merge branch '1.7' into 1.8

accumulo git commit: ACCUMULO-4602 Deleted AssignmentThreadsIT

accumulo git commit: ACCUMULO-4654 Continue balancing even with pending migrations

accumulo git commit: ACCUMULO-4061 Added code to determine Accumulo version from running tserver

6 matches

Site Navigation

Mail list logo

Footer information