date:20230710

[accumulo] 01/01: Merge branch 'main' into elasticity

2023-07-10 Thread dlmarion

This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 406552e46df4412cf8f4f52962446773a9eabee5
Merge: 32f5b4cc69 d4846d407e
Author: Dave Marion 
AuthorDate: Mon Jul 10 17:12:36 2023 +

Merge branch 'main' into elasticity

 .../hadoopImpl/mapred/AccumuloRecordWriter.java|   2 +-
 .../hadoopImpl/mapreduce/AccumuloRecordWriter.java |   2 +-
 .../manager/state/LoggingTabletStateStore.java |   6 +
 .../server/manager/state/MetaDataStateStore.java   |  15 +-
 .../server/manager/state/RootTabletStateStore.java |   5 +-
 .../server/manager/state/TabletStateStore.java |  11 +-
 .../server/manager/state/ZooTabletStateStore.java  |  10 +-
 .../manager/state/ZooTabletStateStoreTest.java |   3 +-
 .../accumulo/manager/TabletGroupWatcher.java   |  47 ++-
 .../apache/accumulo/manager/state/TableCounts.java |   9 ++
 .../apache/accumulo/manager/state/TableStats.java  |   6 +
 .../manager/tserverOps/ShutdownTServer.java|   3 +
 .../org/apache/accumulo/tserver/TabletServer.java  |   1 +
 .../accumulo/tserver/UnloadTabletHandler.java  |   1 +
 .../accumulo/tserver/session/ScanSession.java  |   4 +
 .../apache/accumulo/tserver/session/Session.java   |   6 +
 .../accumulo/tserver/session/SessionManager.java   |  83 ++-
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  14 +-
 .../test/functional/ManagerAssignmentIT.java   | 153 +
 test/src/main/resources/log4j2-test.properties |   3 +
 20 files changed, 335 insertions(+), 49 deletions(-)

diff --cc 
server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index 0189b9294a,acfb363d2b..9f45e36228
--- 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@@ -24,9 -24,9 +24,10 @@@ import java.util.Map
  import java.util.concurrent.TimeUnit;
  
  import org.apache.accumulo.core.logging.TabletLogger;
 +import org.apache.accumulo.core.manager.state.TabletManagement;
  import org.apache.accumulo.core.metadata.TServerInstance;
 -import org.apache.accumulo.core.metadata.TabletLocationState;
+ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
 +import org.apache.accumulo.core.metadata.schema.TabletMetadata;
  import org.apache.hadoop.fs.Path;
  
  import com.google.common.net.HostAndPort;
diff --cc 
server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
index 9874928885,8c7bc888eb..dedffefd14
--- 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
@@@ -19,24 -19,33 +19,28 @@@
  package org.apache.accumulo.server.manager.state;
  
  import java.util.Collection;
 -import java.util.List;
 -import java.util.Map;
  
  import org.apache.accumulo.core.clientImpl.ClientContext;
 +import org.apache.accumulo.core.manager.state.TabletManagement;
  import org.apache.accumulo.core.metadata.MetadataTable;
 -import org.apache.accumulo.core.metadata.TServerInstance;
 -import org.apache.accumulo.core.metadata.TabletLocationState;
  import org.apache.accumulo.core.metadata.schema.Ample;
 +import 
org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult.Status;
+ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
 -import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
  import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 -import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
 -import org.apache.accumulo.core.tabletserver.log.LogEntry;
 -import org.apache.accumulo.server.util.ManagerMetadataUtil;
 -import org.apache.hadoop.fs.Path;
 +import org.apache.accumulo.core.metadata.schema.TabletMetadata;
  
 -class MetaDataStateStore implements TabletStateStore {
 +class MetaDataStateStore extends AbstractTabletStateStore implements 
TabletStateStore {
  
protected final ClientContext context;
protected final CurrentState state;
private final String targetTableName;
private final Ample ample;
+   private final DataLevel level;
  
-   protected MetaDataStateStore(ClientContext context, CurrentState state, 
String targetTableName) {
+   protected MetaDataStateStore(DataLevel level, ClientContext context, 
CurrentState state,
+   String targetTableName) {
 +super(context);
+ this.level = level;
  this.context = context;
  this.state = state;
  this.ample = context.getAmple();
diff --cc 
server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
index e88acb1946,98123e1c2f..034940fb0b
---

[accumulo] branch elasticity updated (32f5b4cc69 -> 406552e46d)

2023-07-10 Thread dlmarion

This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a change to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git


from 32f5b4cc69 Merge branch 'main' into elasticity
 add de02f025ce ensures sessions are always cleaned up (#3569)
 add 767d28d99a avoids calling toString on Text for isEmpty (#3572)
 add 20cecbe8b9 logs info when a tablet consistency check goes from bad to 
good (#3574)
 add 9bbf42f4dc Fix TabletGroupWatcher shutdown order (#3571)
 add d4846d407e Merge branch '2.1'
 new 406552e46d Merge branch 'main' into elasticity

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../hadoopImpl/mapred/AccumuloRecordWriter.java|   2 +-
 .../hadoopImpl/mapreduce/AccumuloRecordWriter.java |   2 +-
 .../manager/state/LoggingTabletStateStore.java |   6 +
 .../server/manager/state/MetaDataStateStore.java   |  15 +-
 .../server/manager/state/RootTabletStateStore.java |   5 +-
 .../server/manager/state/TabletStateStore.java |  11 +-
 .../server/manager/state/ZooTabletStateStore.java  |  10 +-
 .../manager/state/ZooTabletStateStoreTest.java |   3 +-
 .../accumulo/manager/TabletGroupWatcher.java   |  47 ++-
 .../apache/accumulo/manager/state/TableCounts.java |   9 ++
 .../apache/accumulo/manager/state/TableStats.java  |   6 +
 .../manager/tserverOps/ShutdownTServer.java|   3 +
 .../org/apache/accumulo/tserver/TabletServer.java  |   1 +
 .../accumulo/tserver/UnloadTabletHandler.java  |   1 +
 .../accumulo/tserver/session/ScanSession.java  |   4 +
 .../apache/accumulo/tserver/session/Session.java   |   6 +
 .../accumulo/tserver/session/SessionManager.java   |  83 ++-
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  14 +-
 .../test/functional/ManagerAssignmentIT.java   | 153 +
 test/src/main/resources/log4j2-test.properties |   3 +
 20 files changed, 335 insertions(+), 49 deletions(-)

[accumulo] branch main updated (ffe0018b51 -> d4846d407e)

2023-07-10 Thread dlmarion

This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


from ffe0018b51 fix Wait.waitFor usage errors after merge update
 add de02f025ce ensures sessions are always cleaned up (#3569)
 add 767d28d99a avoids calling toString on Text for isEmpty (#3572)
 add 20cecbe8b9 logs info when a tablet consistency check goes from bad to 
good (#3574)
 add 9bbf42f4dc Fix TabletGroupWatcher shutdown order (#3571)
 new d4846d407e Merge branch '2.1'

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../hadoopImpl/mapred/AccumuloRecordWriter.java|   2 +-
 .../hadoopImpl/mapreduce/AccumuloRecordWriter.java |   2 +-
 .../manager/state/LoggingTabletStateStore.java |   6 +
 .../server/manager/state/MetaDataStateStore.java   |  15 +-
 .../server/manager/state/RootTabletStateStore.java |   5 +-
 .../server/manager/state/TabletStateStore.java |  11 +-
 .../server/manager/state/ZooTabletStateStore.java  |  10 +-
 .../manager/state/RootTabletStateStoreTest.java|   3 +-
 .../accumulo/manager/TabletGroupWatcher.java   |  49 +-
 .../apache/accumulo/manager/state/TableCounts.java |   9 ++
 .../apache/accumulo/manager/state/TableStats.java  |   6 +
 .../manager/tserverOps/ShutdownTServer.java|   3 +
 .../org/apache/accumulo/tserver/TabletServer.java  |   1 +
 .../accumulo/tserver/UnloadTabletHandler.java  |   1 +
 .../accumulo/tserver/session/ScanSession.java  |   4 +
 .../apache/accumulo/tserver/session/Session.java   |   6 +
 .../accumulo/tserver/session/SessionManager.java   |  83 ++
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  14 +-
 .../test/functional/ManagerAssignmentIT.java   | 167 +
 test/src/main/resources/log4j2-test.properties |   3 +
 20 files changed, 351 insertions(+), 49 deletions(-)

[accumulo] 01/01: Merge branch '2.1'

2023-07-10 Thread dlmarion

This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit d4846d407e5b28482394e2c0baa16932ae35e086
Merge: ffe0018b51 9bbf42f4dc
Author: Dave Marion 
AuthorDate: Mon Jul 10 16:13:51 2023 +

Merge branch '2.1'

 .../hadoopImpl/mapred/AccumuloRecordWriter.java|   2 +-
 .../hadoopImpl/mapreduce/AccumuloRecordWriter.java |   2 +-
 .../manager/state/LoggingTabletStateStore.java |   6 +
 .../server/manager/state/MetaDataStateStore.java   |  15 +-
 .../server/manager/state/RootTabletStateStore.java |   5 +-
 .../server/manager/state/TabletStateStore.java |  11 +-
 .../server/manager/state/ZooTabletStateStore.java  |  10 +-
 .../manager/state/RootTabletStateStoreTest.java|   3 +-
 .../accumulo/manager/TabletGroupWatcher.java   |  49 +-
 .../apache/accumulo/manager/state/TableCounts.java |   9 ++
 .../apache/accumulo/manager/state/TableStats.java  |   6 +
 .../manager/tserverOps/ShutdownTServer.java|   3 +
 .../org/apache/accumulo/tserver/TabletServer.java  |   1 +
 .../accumulo/tserver/UnloadTabletHandler.java  |   1 +
 .../accumulo/tserver/session/ScanSession.java  |   4 +
 .../apache/accumulo/tserver/session/Session.java   |   6 +
 .../accumulo/tserver/session/SessionManager.java   |  83 ++
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  14 +-
 .../test/functional/ManagerAssignmentIT.java   | 167 +
 test/src/main/resources/log4j2-test.properties |   3 +
 20 files changed, 351 insertions(+), 49 deletions(-)

diff --cc 
server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index b5b16e61d4,84c62254ca..acfb363d2b
--- 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@@ -26,10 -26,10 +26,11 @@@ import java.util.concurrent.TimeUnit
  import org.apache.accumulo.core.logging.TabletLogger;
  import org.apache.accumulo.core.metadata.TServerInstance;
  import org.apache.accumulo.core.metadata.TabletLocationState;
+ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
 -import org.apache.accumulo.core.util.HostAndPort;
  import org.apache.hadoop.fs.Path;
  
 +import com.google.common.net.HostAndPort;
 +
  /**
   * Wraps a tablet state store and logs important events.
   */
diff --cc 
server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
index 6b31568c9e,398e67c112..a2b28e1d71
--- 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
@@@ -28,8 -28,8 +28,9 @@@ import org.apache.accumulo.core.clientI
  import org.apache.accumulo.core.metadata.RootTable;
  import org.apache.accumulo.core.metadata.TServerInstance;
  import org.apache.accumulo.core.metadata.TabletLocationState;
 +import 
org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException;
  import org.apache.accumulo.core.metadata.schema.Ample;
+ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
  import org.apache.accumulo.core.metadata.schema.Ample.ReadConsistency;
  import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
  import org.apache.accumulo.core.metadata.schema.TabletMetadata;
diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
index 7f32b7bfda,f0f8a5de2a..75f1c5f99b
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
@@@ -18,8 -18,11 +18,11 @@@
   */
  package org.apache.accumulo.tserver.session;
  
+ import static java.util.concurrent.TimeUnit.MILLISECONDS;
+ import static java.util.concurrent.TimeUnit.MINUTES;
+ import static java.util.concurrent.TimeUnit.SECONDS;
 +import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
  
 -import java.security.SecureRandom;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.HashSet;
@@@ -39,10 -45,11 +45,11 @@@ import org.apache.accumulo.core.conf.Pr
  import org.apache.accumulo.core.data.Column;
  import org.apache.accumulo.core.data.TableId;
  import org.apache.accumulo.core.dataImpl.thrift.MultiScanResult;
 -import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
 -import org.apache.accumulo.core.tabletserver.thrift.ScanState;
 -import org.apache.accumulo.core.tabletserver.thrift.ScanType;
 +import org.apache.accumulo.core.tabletscan.thrift.ActiveScan;
 +import org.apache.accumulo.core.tabletscan.thrift.ScanState;
 +import org.apache.accumulo.core.tabletscan.thrift.ScanType;
  import org.apache.accumulo.core.util.MapCounter;
+ import

[accumulo] branch 2.1 updated: Fix TabletGroupWatcher shutdown order (#3571)

2023-07-10 Thread dlmarion

This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/2.1 by this push:
 new 9bbf42f4dc Fix TabletGroupWatcher shutdown order (#3571)
9bbf42f4dc is described below

commit 9bbf42f4dc3ebcf21d70c837f577c60708d3d2de
Author: Dave Marion 
AuthorDate: Mon Jul 10 11:40:12 2023 -0400

Fix TabletGroupWatcher shutdown order (#3571)

Modified TabletGroupWatcher such that when shutting
down all TabletServers it unloads user tables before the
metadata table and the metadata table before the root
table.
---
 .../manager/state/LoggingTabletStateStore.java |   6 +
 .../server/manager/state/MetaDataStateStore.java   |  15 +-
 .../server/manager/state/RootTabletStateStore.java |   5 +-
 .../server/manager/state/TabletStateStore.java |  11 +-
 .../server/manager/state/ZooTabletStateStore.java  |  10 +-
 .../manager/state/RootTabletStateStoreTest.java|   3 +-
 .../accumulo/manager/TabletGroupWatcher.java   |  49 +-
 .../apache/accumulo/manager/state/TableCounts.java |   9 ++
 .../apache/accumulo/manager/state/TableStats.java  |   6 +
 .../manager/tserverOps/ShutdownTServer.java|   3 +
 .../org/apache/accumulo/tserver/TabletServer.java  |   1 +
 .../accumulo/tserver/UnloadTabletHandler.java  |   1 +
 .../org/apache/accumulo/tserver/tablet/Tablet.java |   1 +
 .../test/functional/ManagerAssignmentIT.java   | 166 +
 test/src/main/resources/log4j2-test.properties |   3 +
 15 files changed, 276 insertions(+), 13 deletions(-)

diff --git 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index 6ef48fdc07..84c62254ca 100644
--- 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.accumulo.core.logging.TabletLogger;
 import org.apache.accumulo.core.metadata.TServerInstance;
 import org.apache.accumulo.core.metadata.TabletLocationState;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
 import org.apache.accumulo.core.util.HostAndPort;
 import org.apache.hadoop.fs.Path;
 
@@ -40,6 +41,11 @@ class LoggingTabletStateStore implements TabletStateStore {
 this.wrapped = tss;
   }
 
+  @Override
+  public DataLevel getLevel() {
+return wrapped.getLevel();
+  }
+
   @Override
   public String name() {
 return wrapped.name();
diff --git 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
index bb27b32cda..8c7bc888eb 100644
--- 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
@@ -27,6 +27,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
 import org.apache.accumulo.core.metadata.TServerInstance;
 import org.apache.accumulo.core.metadata.TabletLocationState;
 import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
 import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
@@ -40,16 +41,24 @@ class MetaDataStateStore implements TabletStateStore {
   protected final CurrentState state;
   private final String targetTableName;
   private final Ample ample;
+  private final DataLevel level;
 
-  protected MetaDataStateStore(ClientContext context, CurrentState state, 
String targetTableName) {
+  protected MetaDataStateStore(DataLevel level, ClientContext context, 
CurrentState state,
+  String targetTableName) {
+this.level = level;
 this.context = context;
 this.state = state;
 this.ample = context.getAmple();
 this.targetTableName = targetTableName;
   }
 
-  MetaDataStateStore(ClientContext context, CurrentState state) {
-this(context, state, MetadataTable.NAME);
+  MetaDataStateStore(DataLevel level, ClientContext context, CurrentState 
state) {
+this(level, context, state, MetadataTable.NAME);
+  }
+
+  @Override
+  public DataLevel getLevel() {
+return level;
   }
 
   @Override
diff --git 
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
 
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java
index 9acd1cc48a..98123e1c2f 100644
---

[accumulo] branch 2.1 updated: logs info when a tablet consistency check goes from bad to good (#3574)

2023-07-10 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/2.1 by this push:
 new 20cecbe8b9 logs info when a tablet consistency check goes from bad to 
good (#3574)
20cecbe8b9 is described below

commit 20cecbe8b9c8b55d17f341dfd64f5b42f7f79705
Author: Keith Turner 
AuthorDate: Mon Jul 10 11:29:05 2023 -0400

logs info when a tablet consistency check goes from bad to good (#3574)

When the tablet server does a periodic consistency check on a tablet
it logs an error when there is a mismatch.  Later if there is no
longer a mismatch, then this commit changes the code to log an info.

related to #3537 but does not fix that issue
---
 .../java/org/apache/accumulo/tserver/tablet/Tablet.java | 13 +++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 489dbd0700..1b818d8759 100644
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1114,6 +1114,8 @@ public class Tablet extends TabletBase {
 }
   }
 
+  private boolean loggedErrorForTabletComparison = false;
+
   /**
* Checks that tablet metadata from the metadata table matches what this 
tablet has in memory. The
* caller of this method must acquire the updateCounter parameter before 
acquiring the
@@ -1162,10 +1164,17 @@ public class Tablet extends TabletBase {
   } else {
 log.error("Data files in {} differ from in-memory data {} {} {} {}", 
extent,
 tabletMetadata.getFilesMap(), dataFileSizes, updateCounter, 
latestCount);
+loggedErrorForTabletComparison = true;
   }
 } else {
-  log.trace("AMCC Tablet {} files in memory are same as in metadata table 
{}",
-  tabletMetadata.getExtent(), updateCounter);
+  if (loggedErrorForTabletComparison) {
+log.info("AMCC Tablet {} files in memory are now same as in metadata 
table {}",
+tabletMetadata.getExtent(), updateCounter);
+loggedErrorForTabletComparison = false;
+  } else {
+log.trace("AMCC Tablet {} files in memory are same as in metadata 
table {}",
+tabletMetadata.getExtent(), updateCounter);
+  }
 }
   }

[accumulo] branch 2.1 updated: avoids calling toString on Text for isEmpty (#3572)

2023-07-10 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/2.1 by this push:
 new 767d28d99a avoids calling toString on Text for isEmpty (#3572)
767d28d99a is described below

commit 767d28d99aee1d460d05f82065346c5451a68b0c
Author: Keith Turner 
AuthorDate: Mon Jul 10 11:28:36 2023 -0400

avoids calling toString on Text for isEmpty (#3572)

In profiling of an Accumulo map reduce job it was noticed that a good
bit of time was spent calling toString on a Text object to check if it
was empty. This commit avoids the toString() call and instead checks the
text length.
---
 .../org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java| 2 +-
 .../org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java | 2 +-
 .../org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java | 2 +-
 .../org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git 
a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
 
b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index 10d517c361..6dbecaee6b 100644
--- 
a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@ -425,7 +425,7 @@ public class AccumuloOutputFormat implements 
OutputFormat {
  */
 @Override
 public void write(Text table, Mutation mutation) throws IOException {
-  if (table == null || table.toString().isEmpty()) {
+  if (table == null || table.getLength() == 0) {
 table = this.defaultTableName;
   }
 
diff --git 
a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
 
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 3c2193b3d3..c39c1013c1 100644
--- 
a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@ -426,7 +426,7 @@ public class AccumuloOutputFormat extends 
OutputFormat {
  */
 @Override
 public void write(Text table, Mutation mutation) throws IOException {
-  if (table == null || table.toString().isEmpty()) {
+  if (table == null || table.getLength() == 0) {
 table = this.defaultTableName;
   }
 
diff --git 
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
 
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
index ac012a1cf0..a29caf9c7f 100644
--- 
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
+++ 
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
@@ -89,7 +89,7 @@ public class AccumuloRecordWriter implements 
RecordWriter {
*/
   @Override
   public void write(Text table, Mutation mutation) throws IOException {
-if (table == null || table.toString().isEmpty()) {
+if (table == null || table.getLength() == 0) {
   table = this.defaultTableName;
 }
 
diff --git 
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
 
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
index 1399d8d0b7..fb21102ff2 100644
--- 
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
+++ 
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
@@ -90,7 +90,7 @@ public class AccumuloRecordWriter extends 
RecordWriter {
*/
   @Override
   public void write(Text table, Mutation mutation) throws IOException {
-if (table == null || table.toString().isEmpty()) {
+if (table == null || table.getLength() == 0) {
   table = this.defaultTableName;
 }

[accumulo] branch 2.1 updated: ensures sessions are always cleaned up (#3569)

2023-07-10 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/2.1 by this push:
 new de02f025ce ensures sessions are always cleaned up (#3569)
de02f025ce is described below

commit de02f025ce5b25acfb86dbad9e9b96c7115d653f
Author: Keith Turner 
AuthorDate: Mon Jul 10 11:27:46 2023 -0400

ensures sessions are always cleaned up (#3569)

This is a potential fix for #3512.  It ensures that when a sesssions
cleanup method returns false that cleanup will be attempted again later.
---
 .../accumulo/tserver/session/ScanSession.java  |  4 ++
 .../apache/accumulo/tserver/session/Session.java   |  6 ++
 .../accumulo/tserver/session/SessionManager.java   | 84 +-
 3 files changed, 62 insertions(+), 32 deletions(-)

diff --git 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
index 3217fe1b8f..0fefcc1327 100644
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
@@ -189,4 +189,8 @@ public abstract class ScanSession extends Session 
implements ScanInfo {
 return true;
   }
 
+  @Override
+  public String toString() {
+return super.toString() + " tableId:" + getTableId();
+  }
 }
diff --git 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
index b1c14ca6e8..6e49833729 100644
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
@@ -49,4 +49,10 @@ public class Session {
   public boolean cleanup() {
 return true;
   }
+
+  @Override
+  public String toString() {
+return getClass().getSimpleName() + " " + state + " startTime:" + 
startTime + " lastAccessTime:"
++ lastAccessTime + " client:" + client;
+  }
 }
diff --git 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
index b8d605ebd8..f0f8a5de2a 100644
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
@@ -18,15 +18,22 @@
  */
 package org.apache.accumulo.tserver.session;
 
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ScheduledFuture;
@@ -42,6 +49,7 @@ import 
org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
 import org.apache.accumulo.core.tabletserver.thrift.ScanState;
 import org.apache.accumulo.core.tabletserver.thrift.ScanType;
 import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Retry;
 import org.apache.accumulo.core.util.threads.ThreadPools;
 import org.apache.accumulo.server.ServerContext;
 import org.apache.accumulo.tserver.scan.ScanRunState;
@@ -61,7 +69,7 @@ public class SessionManager {
   private final ConcurrentMap sessions = new 
ConcurrentHashMap<>();
   private final long maxIdle;
   private final long maxUpdateIdle;
-  private final List idleSessions = new ArrayList<>();
+  private final BlockingQueue deferredCleanupQueue = new 
ArrayBlockingQueue<>(5000);
   private final Long expiredSessionMarker = (long) -1;
   private final AccumuloConfiguration aconf;
   private final ServerContext ctx;
@@ -209,15 +217,39 @@ public class SessionManager {
   }
 
   if (doCleanup) {
-session.cleanup();
+cleanup(session);
   }
 }
 
 return session;
   }
 
+  private void cleanup(Session session) {
+if (!session.cleanup()) {
+  var retry = Retry.builder().infiniteRetries().retryAfter(25, 
MILLISECONDS)
+  .incrementBy(25, MILLISECONDS).maxWait(5, SECONDS).backOffFactor(1.5)
+  .logInterval(1, MINUTES).createRetry();
+
+  while (!deferredCleanupQueue.offer(session)) {
+if (session.cleanup()) {
+  break;
+}
+
+try {
+  retry.waitForNextAttempt(log, "Unable to cleanup session or

[accumulo] 01/01: Merge branch 'main' into elasticity

[accumulo] branch elasticity updated (32f5b4cc69 -> 406552e46d)

[accumulo] branch main updated (ffe0018b51 -> d4846d407e)

[accumulo] 01/01: Merge branch '2.1'

[accumulo] branch 2.1 updated: Fix TabletGroupWatcher shutdown order (#3571)

[accumulo] branch 2.1 updated: logs info when a tablet consistency check goes from bad to good (#3574)

[accumulo] branch 2.1 updated: avoids calling toString on Text for isEmpty (#3572)

[accumulo] branch 2.1 updated: ensures sessions are always cleaned up (#3569)

8 matches

Site Navigation

Mail list logo

Footer information