(accumulo) branch elasticity updated: ignores failures to set future location (#4579)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 14efea2e75 ignores failures to set future location (#4579)
14efea2e75 is described below
commit 14efea2e757ce91d0ff732c51c18a03950d9a719
Author: Keith Turner
AuthorDate: Wed May 22 17:43:46 2024 -0400
ignores failures to set future location (#4579)
When TabletGroupWatcher failed to set a future location it would
still ask the tablet server to load the tablet. The tablet server
would get the request and fail causing noise in its logs. This
change avoids uneeded work and noise in the logs.
---
.../server/manager/state/AbstractTabletStateStore.java | 7 ++-
.../server/manager/state/LoggingTabletStateStore.java | 13 ++---
.../accumulo/server/manager/state/TabletStateStore.java | 6 +-
.../accumulo/server/manager/state/ZooTabletStateStore.java | 6 --
.../org/apache/accumulo/manager/TabletGroupWatcher.java | 7 ++-
5 files changed, 31 insertions(+), 8 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
index e6c78ace50..56f80eeae9 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
@@ -18,10 +18,12 @@
*/
package org.apache.accumulo.server.manager.state;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.TServerInstance;
@@ -79,7 +81,7 @@ public abstract class AbstractTabletStateStore implements
TabletStateStore {
}
@Override
- public void setFutureLocations(Collection assignments)
+ public Set setFutureLocations(Collection assignments)
throws DistributedStoreException {
try (var tabletsMutator = ample.conditionallyMutateTablets()) {
for (Assignment assignment : assignments) {
@@ -94,14 +96,17 @@ public abstract class AbstractTabletStateStore implements
TabletStateStore {
}
Map results = tabletsMutator.process();
+ List failed = new ArrayList<>();
for (Entry entry : results.entrySet()) {
if (entry.getValue().getStatus() != Status.ACCEPTED) {
LOG.debug("Likely concurrent FATE operation prevented setting future
location for {}, "
+ "Manager will retry soon.", entry.getKey());
+ failed.add(entry.getKey());
}
}
+ return Set.copyOf(failed);
} catch (RuntimeException ex) {
throw new DistributedStoreException(ex);
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index ec93787abb..0c60389848 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@ -21,8 +21,10 @@ package org.apache.accumulo.server.manager.state;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.manager.state.TabletManagement;
import org.apache.accumulo.core.metadata.TServerInstance;
@@ -61,10 +63,15 @@ class LoggingTabletStateStore implements TabletStateStore {
}
@Override
- public void setFutureLocations(Collection assignments)
+ public Set setFutureLocations(Collection assignments)
throws DistributedStoreException {
-wrapped.setFutureLocations(assignments);
-assignments.forEach(assignment -> TabletLogger.assigned(assignment.tablet,
assignment.server));
+var failures = wrapped.setFutureLocations(assignments);
+assignments.forEach(assignment -> {
+ if (!failures.contains(assignment.tablet)) {
+TabletLogger.assigned(assignment.tablet, assignment.server);
+ }
+});
+return failures;
}
@Override
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
index 602409125e..c4f3910d51 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/st
(accumulo) branch elasticity updated: cleans up some todos (#4588)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5f694dfab2 cleans up some todos (#4588)
5f694dfab2 is described below
commit 5f694dfab28fe3c52b2b3349af20723b92375b9e
Author: Keith Turner
AuthorDate: Wed May 22 16:15:24 2024 -0400
cleans up some todos (#4588)
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 5 ++---
.../main/java/org/apache/accumulo/server/AccumuloDataVersion.java| 2 +-
.../src/test/java/org/apache/accumulo/server/ServerContextTest.java | 2 +-
.../manager/compaction/coordinator/CompactionCoordinator.java| 2 +-
test/src/main/java/org/apache/accumulo/test/UnusedWALIT.java | 2 +-
5 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 66c27f9311..17226fcbf8 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,9 +444,8 @@ public enum Property {
"The number of threads used to inspect tablets files to find split
points.", "4.0.0"),
MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE("manager.compaction.major.service.queue.size",
- // ELASTICITY_TODO: It might be good to note that there is a priority
queue per compactor
- // resource group
- "1", PropertyType.COUNT, "The max size of the priority queue.",
"4.0"),
+ "1", PropertyType.COUNT,
+ "The max size of each resource groups compaction job priority queue.",
"4.0"),
SPLIT_PREFIX("split.", null, PropertyType.PREFIX,
"System wide properties related to splitting tablets.", "3.1.0"),
SPLIT_MAXOPEN("split.files.max", "300", PropertyType.COUNT,
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
index 224cbde6af..598b8e48cf 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
@@ -82,7 +82,7 @@ public class AccumuloDataVersion {
return CURRENT_VERSION;
}
- // ELASTICITY_TODO get upgrade working
+ // ELASTICITY_TODO get upgrade working #4587
// public static final Set CAN_RUN =
Set.of(ROOT_TABLET_META_CHANGES,
// REMOVE_DEPRECATIONS_FOR_VERSION_3, METADATA_FILE_JSON_ENCODING,
CURRENT_VERSION);
public static final Set CAN_RUN = Set.of(CURRENT_VERSION);
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
b/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
index d5909ecf99..eac064f88f 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
@@ -134,7 +134,7 @@ public class ServerContextTest {
@Test
public void testCanRun() {
final int oldestSupported = AccumuloDataVersion.oldestUpgradeableVersion();
-// ELASTICITY_TODO basically disable check until upgrade is working.
Should be:
+// ELASTICITY_TODO #4587 basically disable check until upgrade is working.
Should be:
// assertEquals(10, oldestSupported); // make sure it hasn't changed
accidentally
final int currentVersion = AccumuloDataVersion.get();
IntConsumer shouldPass = ServerContext::ensureDataVersionCompatible;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index bce38de18d..7642003985 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -158,7 +158,7 @@ public class CompactionCoordinator
new ConcurrentHashMap<>();
/* Map of group name to last time compactor called to get a compaction job */
- // ELASTICITY_TODO need to clean out groups that are no longer configured..
+ // ELASTICITY_TODO #4403 need to clean out groups that are no longer
configured..
private final Map TIME_COMPACTOR_LAST_CHECKED = new
ConcurrentHashMap<>();
private final ServerContext ctx;
diff --git a/test/src/main/java/org/apache/accumulo/test/UnusedWALIT.java
b/test/src/main/java/org/apache/accumulo/test/UnusedWALIT.java
index 84db1bba50..1923a9debf 100
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 0b0d0d39a415decae13099cd68beac85ef30109d
Merge: 8f18d29f32 9edd74bb06
Author: Keith Turner
AuthorDate: Wed May 22 10:49:34 2024 -0400
Merge branch 'main' into elasticity
.../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 10 +-
.../file/blockfile/cache/lru/LruBlockCacheConfiguration.java | 4
.../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 4 +++-
3 files changed, 12 insertions(+), 6 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
index ab86ac1a58,7ee9dadbf8..2baba46caf
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
@@@ -64,14 -63,15 +64,15 @@@ public final class TinyLfuBlockCache im
private final int maxSize;
private final ScheduledExecutorService statsExecutor =
ThreadPools.getServerThreadPools()
.createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor");
+ private final CacheType type;
public TinyLfuBlockCache(Configuration conf, CacheType type) {
-cache = Caffeine.newBuilder()
+cache =
Caches.getInstance().createNewBuilder(CacheName.TINYLFU_BLOCK_CACHE, false)
.initialCapacity((int) Math.ceil(1.2 * conf.getMaxSize(type) /
conf.getBlockSize()))
-.weigher((String blockName, Block block) -> {
+.recordStats().weigher((String blockName, Block block) -> {
int keyWeight = ClassSize.align(blockName.length()) +
ClassSize.STRING;
return keyWeight + block.weight();
-}).maximumWeight(conf.getMaxSize(type)).recordStats().build();
+}).maximumWeight(conf.getMaxSize(type)).build();
policy = cache.policy().eviction().orElseThrow();
maxSize = (int) Math.min(Integer.MAX_VALUE, policy.getMaximum());
ScheduledFuture future =
statsExecutor.scheduleAtFixedRate(this::logStats, STATS_PERIOD_SEC,
(accumulo) branch elasticity updated (8f18d29f32 -> 0b0d0d39a4)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8f18d29f32 adds toggle for automatically setting lock on mutation (#4584) add 187074c908 log cache type when logging cache stats (#4585) add 9edd74bb06 Merge branch '2.1' new 0b0d0d39a4 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 10 +- .../file/blockfile/cache/lru/LruBlockCacheConfiguration.java | 4 .../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9edd74bb06d180ad2b896a925476771abab9983e Merge: 3a223a0196 187074c908 Author: Keith Turner AuthorDate: Wed May 22 10:47:10 2024 -0400 Merge branch '2.1' .../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 10 +- .../file/blockfile/cache/lru/LruBlockCacheConfiguration.java | 4 .../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-)
(accumulo) branch main updated (3a223a0196 -> 9edd74bb06)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 3a223a0196 Merge branch '2.1' add 187074c908 log cache type when logging cache stats (#4585) new 9edd74bb06 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 10 +- .../file/blockfile/cache/lru/LruBlockCacheConfiguration.java | 4 .../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-)
(accumulo) branch 2.1 updated: log cache type when logging cache stats (#4585)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 187074c908 log cache type when logging cache stats (#4585)
187074c908 is described below
commit 187074c908f3f64181f990e74562bf9c2fa33799
Author: Keith Turner
AuthorDate: Wed May 22 10:46:36 2024 -0400
log cache type when logging cache stats (#4585)
---
.../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 10 +-
.../file/blockfile/cache/lru/LruBlockCacheConfiguration.java | 4
.../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 4 +++-
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
index 0183ebe3bb..dc85350405 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
@@ -577,14 +577,14 @@ public class LruBlockCache extends
SynchronousLoadingBlockCache implements Block
float freeMB = ((float) freeSize) / ((float) (1024 * 1024));
float maxMB = ((float) this.conf.getMaxSize()) / ((float) (1024 * 1024));
log.debug(
-"Cache Stats: Sizes: Total={}MB ({}), Free={}MB ({}), Max={}MB"
+"Cache Stats: {} Sizes: Total={}MB ({}), Free={}MB ({}), Max={}MB"
+ " ({}), Counts: Blocks={}, Access={}, Hit={}, Miss={},
Evictions={},"
+ " Evicted={},Ratios: Hit Ratio={}%, Miss Ratio={}%,
Evicted/Run={},"
+ " Duplicate Reads={}",
-sizeMB, totalSize, freeMB, freeSize, maxMB, this.conf.getMaxSize(),
size(),
-stats.requestCount(), stats.hitCount(), stats.getMissCount(),
stats.getEvictionCount(),
-stats.getEvictedCount(), stats.getHitRatio() * 100,
stats.getMissRatio() * 100,
-stats.evictedPerEviction(), stats.getDuplicateReads());
+conf.getCacheType(), sizeMB, totalSize, freeMB, freeSize, maxMB,
this.conf.getMaxSize(),
+size(), stats.requestCount(), stats.hitCount(), stats.getMissCount(),
+stats.getEvictionCount(), stats.getEvictedCount(), stats.getHitRatio()
* 100,
+stats.getMissRatio() * 100, stats.evictedPerEviction(),
stats.getDuplicateReads());
}
/**
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java
index 0d427504cd..0ce59c993b 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java
@@ -238,6 +238,10 @@ public final class LruBlockCacheConfiguration {
return conf.getMaxSize(type);
}
+ public CacheType getCacheType() {
+return type;
+ }
+
public long getBlockSize() {
return conf.getBlockSize();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
index 46a07682bd..78206c7bd3 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
@@ -63,6 +63,7 @@ public final class TinyLfuBlockCache implements BlockCache {
private final int maxSize;
private final ScheduledExecutorService statsExecutor =
ThreadPools.getServerThreadPools()
.createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor");
+ private final CacheType type;
public TinyLfuBlockCache(Configuration conf, CacheType type) {
cache = Caffeine.newBuilder()
@@ -75,6 +76,7 @@ public final class TinyLfuBlockCache implements BlockCache {
maxSize = (int) Math.min(Integer.MAX_VALUE, policy.getMaximum());
ScheduledFuture future =
statsExecutor.scheduleAtFixedRate(this::logStats, STATS_PERIOD_SEC,
STATS_PERIOD_SEC, SECONDS);
+this.type = type;
ThreadPools.watchNonCriticalScheduledTask(future);
}
@@ -120,7 +122,7 @@ public final class TinyLfuBlockCache implements BlockCache {
double maxMB = ((double) policy.getMaximum()) / ((double) (1024 * 1024));
double sizeMB = ((double) policy.weightedSize().orElse(0)) / ((double)
(1024 * 1024));
double freeMB = maxMB - sizeMB;
-log.debug("Cache Size={}MB, Free={}MB, Max={}MB, Blocks={}", sizeMB,
freeMB, maxMB,
+log.debug("Cache
(accumulo) branch elasticity updated: adds toggle for automatically setting lock on mutation (#4584)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 8f18d29f32 adds toggle for automatically setting lock on mutation
(#4584)
8f18d29f32 is described below
commit 8f18d29f32e678a76a205e36e3acfc5ead2863ae
Author: Keith Turner
AuthorDate: Tue May 21 18:28:30 2024 -0400
adds toggle for automatically setting lock on mutation (#4584)
Merging tables in a table deletes tablets. When tablets were
being deleted they were automatically setting a lock, which
could leave junk in the metadata table. Added a toggle to
turn off setting these on mutations.
---
.../main/java/org/apache/accumulo/core/metadata/schema/Ample.java | 6 ++
.../accumulo/core/metadata/schema/TabletMetadataBuilder.java | 5 +
.../apache/accumulo/core/metadata/schema/TabletMutatorBase.java| 7 +++
.../accumulo/server/metadata/ConditionalTabletMutatorImpl.java | 4 +++-
.../org/apache/accumulo/server/metadata/RootTabletMutatorImpl.java | 4 +++-
.../org/apache/accumulo/server/metadata/TabletMutatorImpl.java | 4 +++-
.../org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java | 4
7 files changed, 31 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 7d7086649a..3c00d70f33 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -421,6 +421,12 @@ public interface Ample {
T setUnSplittable(UnSplittableMetadata unSplittableMeta);
T deleteUnSplittable();
+
+/**
+ * By default the server lock is automatically added to mutations unless
this method is set to
+ * false.
+ */
+T automaticallyPutServerLock(boolean b);
}
interface TabletMutator extends TabletUpdates {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
index 4121ad2005..eb924e7833 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
@@ -303,6 +303,11 @@ public class TabletMetadataBuilder implements
Ample.TabletUpdates>
protected final Mutation mutation;
protected AutoCloseable closeAfterMutate;
protected boolean updatesEnabled = true;
+ protected boolean putServerLock = true;
@SuppressWarnings("unchecked")
private T getThis() {
@@ -360,6 +361,12 @@ public abstract class TabletMutatorBase>
return getThis();
}
+ @Override
+ public T automaticallyPutServerLock(boolean b) {
+putServerLock = b;
+return getThis();
+ }
+
public void setCloseAfterMutate(AutoCloseable closeable) {
this.closeAfterMutate = closeable;
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
index 7c3d9a5859..208fe4954c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
@@ -295,7 +295,9 @@ public class ConditionalTabletMutatorImpl extends
TabletMutatorBase
@Override
public void mutate() {
try {
- this.putZooLock(this.context.getZooKeeperRoot(), lock);
+ if (putServerLock) {
+this.putZooLock(this.context.getZooKeeperRoot(), lock);
+ }
writer.addMutation(getMutation());
if (closeAfterMutate != null) {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
index 710b8779ad..3cb48a68b0 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
@@ -107,6 +107,10 @@ public class DeleteTablets extends ManagerRepo {
}
tabletMutator.deleteAll(tabletMeta.getKeyValues().keySet());
+
+// the entire tablet is being deleted, so do not want to add lock
entry to the tablet
+tabletMutator.automaticallyPutServerLock(false);
+
// if the tablet no longer exists, then it was successful
tabletMutator.submit(Ample.RejectionHandler.acceptAbsentTablet());
submitted++;
(accumulo) branch elasticity updated: lowers time to host ondemand tablets (#4581)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5aca710487 lowers time to host ondemand tablets (#4581)
5aca710487 is described below
commit 5aca710487d4b40bb5a63588707bc9e35c5694b9
Author: Keith Turner
AuthorDate: Tue May 21 18:28:16 2024 -0400
lowers time to host ondemand tablets (#4581)
This change lowers the time it takes to host ondemand tablets
by moving this functionality into TabletGroupWatcher.
The client RPC thread processing the hosting request can now
directly call a function in TGW that will immediately start on
the work of hosting the tablets.
Updated SplitMillionIT to request hosting of 200 tablets all
at once instead of one by one. This was done by using a
BatchScanner instead of lots of scanners.
---
.../java/org/apache/accumulo/manager/Manager.java | 10 ++
.../manager/ManagerClientServiceHandler.java | 49 +-
.../accumulo/manager/TabletGroupWatcher.java | 102 -
.../accumulo/test/functional/SplitMillionIT.java | 42 +
4 files changed, 118 insertions(+), 85 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 244a2696ab..404fe0ba7c 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -549,6 +549,16 @@ public class Manager extends AbstractServer
return compactionCoordinator;
}
+ public void hostOndemand(List extents) {
+extents.forEach(e -> Preconditions.checkArgument(DataLevel.of(e.tableId())
== DataLevel.USER));
+
+for (var watcher : watchers) {
+ if (watcher.getLevel() == DataLevel.USER) {
+watcher.hostOndemand(extents);
+ }
+}
+ }
+
private class MigrationCleanupThread implements Runnable {
@Override
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
index e70e151ace..abfb6675af 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
@@ -33,13 +33,11 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
-import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.clientImpl.AuthenticationTokenIdentifier;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.DelegationTokenConfigSerializer;
@@ -69,7 +67,6 @@ import
org.apache.accumulo.core.manager.thrift.TabletLoadState;
import org.apache.accumulo.core.manager.thrift.ThriftPropertyException;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult.Status;
import org.apache.accumulo.core.metadata.schema.TabletDeletedException;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
@@ -92,15 +89,15 @@ import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.slf4j.Logger;
+import com.google.common.collect.Lists;
+
public class ManagerClientServiceHandler implements ManagerClientService.Iface
{
private static final Logger log = Manager.log;
private final Manager manager;
- private final Set hostingRequestInProgress;
protected ManagerClientServiceHandler(Manager manager) {
this.manager = manager;
-this.hostingRequestInProgress = new ConcurrentSkipListSet<>();
}
@Override
@@ -611,51 +608,11 @@ public class ManagerClientServiceHandler implements
ManagerClientService.Iface {
manager.mustBeOnline(tableId);
-final List success = new ArrayList<>();
-final List inProgress = new ArrayList<>();
-extents.forEach(e -> {
- KeyExtent ke = KeyExtent.fromThrift(e);
- if (hostingRequestInProgress.add(ke)) {
-log.info("Tablet hosting requested for: {} ", KeyExtent.fromThrift(e));
-inProgress.add(ke);
- } else {
-log.trace("Ignoring hosting request because another thread is
curre
(accumulo) branch elasticity updated: fixes timeout in CompactionPriorityQueueMetricsIT (#4583)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new aa85bf2b0e fixes timeout in CompactionPriorityQueueMetricsIT (#4583)
aa85bf2b0e is described below
commit aa85bf2b0ec57c7bc53cabc293be993d648c15a6
Author: Keith Turner
AuthorDate: Tue May 21 15:27:13 2024 -0400
fixes timeout in CompactionPriorityQueueMetricsIT (#4583)
CompactionPriorityQueueMetricsIT was waiting up to 30s
for a dead compactor process to be absent in zookeeper.
The default zookeeper timeout is 30s, so there was a good
chance it would timeout. Modified the zookeeper timeout
in the test to 10s rather then increase the time to wait
for the ZK entry to be absent.
---
.../test/compaction/CompactionPriorityQueueMetricsIT.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
index 7af245fde3..40f6ddc107 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
@@ -152,7 +152,9 @@ public class CompactionPriorityQueueMetricsIT extends
SharedMiniClusterBase {
@AfterEach
public void teardownMetricsTest() throws Exception {
shutdownTailer.set(true);
-metricsTailer.join();
+if (metricsTailer != null) {
+ metricsTailer.join();
+}
}
private String getDir(String testName) throws Exception {
@@ -197,6 +199,10 @@ public class CompactionPriorityQueueMetricsIT extends
SharedMiniClusterBase {
cfg.setProperty(Property.MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE,
"6");
cfg.getClusterServerConfiguration().addCompactorResourceGroup(QUEUE1, 0);
+ // This test waits for dead compactors to be absent in zookeeper. The
following setting will
+ // make entries in ZK related to dead compactor processes expire sooner.
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT.getKey(), "10");
+
// use raw local file system
conf.set("fs.file.impl", RawLocalFileSystem.class.getName());
// Tell the server processes to use a StatsDMeterRegistry that will be
configured
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit df9e9ad136e00d467f04dec381c5a3e36a2bdc7b Merge: f9ab9c9d95 3a223a0196 Author: Keith Turner AuthorDate: Tue May 21 12:36:43 2024 -0400 Merge branch 'main' into elasticity .../src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java | 3 +++ 1 file changed, 3 insertions(+)
(accumulo) branch elasticity updated (f9ab9c9d95 -> df9e9ad136)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from f9ab9c9d95 fixes AmpleConditionalWriterIT test failure (#4582) add 4d3a4d51b5 ensures instrumented cache is recording stats (#4552) add 3a223a0196 Merge branch '2.1' new df9e9ad136 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java | 3 +++ 1 file changed, 3 insertions(+)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3a223a01965bb590e96adfdc7018b8a630619872
Merge: 91a2ca2349 4d3a4d51b5
Author: Keith Turner
AuthorDate: Tue May 21 12:35:19 2024 -0400
Merge branch '2.1'
.../src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java | 3 +++
1 file changed, 3 insertions(+)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
index 37ddfab028,771def8e4f..e555b8383e
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
@@@ -25,8 -23,9 +25,9 @@@ import org.apache.accumulo.core.metadat
import org.apache.accumulo.core.metrics.MetricsProducer;
import com.github.benmanes.caffeine.cache.LoadingCache;
+ import com.google.common.base.Preconditions;
-import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.FunctionCounter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.binder.cache.CaffeineCacheMetrics;
@@@ -46,8 -45,10 +47,10 @@@ public class ScanServerMetrics implemen
public void registerMetrics(MeterRegistry registry) {
reservationTimer =
Timer.builder(MetricsProducer.METRICS_SCAN_RESERVATION_TIMER)
.description("Time to reserve a tablets files for
scan").register(registry);
-busyTimeoutCount = Counter.builder(METRICS_SCAN_BUSY_TIMEOUT_COUNTER)
+FunctionCounter.builder(METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
busyTimeoutCount, AtomicLong::get)
.description("The number of scans where a busy timeout
happened").register(registry);
+ Preconditions.checkState(tabletMetadataCache.policy().isRecordingStats(),
+ "Attempted to instrument cache that is not recording stats.");
CaffeineCacheMetrics.monitor(registry, tabletMetadataCache,
METRICS_SCAN_TABLET_METADATA_CACHE);
}
(accumulo) branch main updated (91a2ca2349 -> 3a223a0196)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 91a2ca2349 Merge branch '2.1' add 4d3a4d51b5 ensures instrumented cache is recording stats (#4552) new 3a223a0196 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java | 3 +++ 1 file changed, 3 insertions(+)
(accumulo) branch 2.1 updated: ensures instrumented cache is recording stats (#4552)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 4d3a4d51b5 ensures instrumented cache is recording stats (#4552)
4d3a4d51b5 is described below
commit 4d3a4d51b506f779ac38d12e3a344fe8bef16560
Author: Keith Turner
AuthorDate: Tue May 21 12:34:56 2024 -0400
ensures instrumented cache is recording stats (#4552)
---
.../src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
index 1a516b597b..771def8e4f 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
@@ -23,6 +23,7 @@ import
org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metrics.MetricsProducer;
import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.base.Preconditions;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
@@ -46,6 +47,8 @@ public class ScanServerMetrics implements MetricsProducer {
.description("Time to reserve a tablets files for
scan").register(registry);
busyTimeoutCount = Counter.builder(METRICS_SCAN_BUSY_TIMEOUT_COUNTER)
.description("The number of scans where a busy timeout
happened").register(registry);
+Preconditions.checkState(tabletMetadataCache.policy().isRecordingStats(),
+"Attempted to instrument cache that is not recording stats.");
CaffeineCacheMetrics.monitor(registry, tabletMetadataCache,
METRICS_SCAN_TABLET_METADATA_CACHE);
}
(accumulo) branch elasticity updated: fixes AmpleConditionalWriterIT test failure (#4582)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f9ab9c9d95 fixes AmpleConditionalWriterIT test failure (#4582)
f9ab9c9d95 is described below
commit f9ab9c9d950f973c423169f24e141db11cbabb64
Author: Keith Turner
AuthorDate: Tue May 21 12:00:07 2024 -0400
fixes AmpleConditionalWriterIT test failure (#4582)
---
.../org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
index 1f6627a7ec..3c51f2db47 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
@@ -672,7 +672,7 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
Collection paths, int compJobs, long selTime) {
String filesJsonArray = GSON.get().toJson(paths);
return ("{'fateId':'" + fateId + "','selAll':" + selAll + ",'compJobs':" +
compJobs
-+ ",'selTime':" + selTime + ",'files':" + filesJsonArray +
"}").replace('\'', '\"');
++ ",'selTimeNanos':" + selTime + ",'files':" + filesJsonArray +
"}").replace('\'', '\"');
}
@Test
(accumulo) branch elasticity updated: fixes deleting tablet suspension (#4575)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e98f15b5fd fixes deleting tablet suspension (#4575)
e98f15b5fd is described below
commit e98f15b5fd282ee60d2d695531ee16b89014d459
Author: Keith Turner
AuthorDate: Mon May 20 16:17:45 2024 -0400
fixes deleting tablet suspension (#4575)
Tablet suspension was failing to delete becasue the value set on the
conditional mutation did not match the format of what was stored in the
metadata table. Fixed this issue and added some unit test.
---
.../org/apache/accumulo/core/data/Condition.java | 15 ++
.../accumulo/core/metadata/SuspendingTServer.java | 4 ++
.../core/metadata/SuspendingTServerTest.java | 63 ++
.../metadata/ConditionalTabletMutatorImpl.java | 3 +-
4 files changed, 83 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
index deadc0e069..dab5ee24e5 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Condition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Condition.java
@@ -209,6 +209,21 @@ public class Condition {
return this;
}
+ /**
+ * This method sets the expected value of a column. In order for the
condition to pass the column
+ * must exist and have this value. If a value is not set, then the column
must be absent for the
+ * condition to pass. See {@link #setValue(byte[])}.
+ *
+ * @param value value
+ * @return this condition
+ * @throws IllegalArgumentException if value is null
+ * @since 4.0.0
+ */
+ public Condition setValue(Value value) {
+checkArgument(value != null, "value is null");
+return setValue(value.get());
+ }
+
/**
* Gets the value of this condition.
*
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
b/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
index e481369a21..e45635fc80 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
@@ -49,6 +49,10 @@ public class SuspendingTServer {
return new Value(tServer.getHostPort() + "|" + suspensionTime.getMillis());
}
+ public Value toValue() {
+return new Value(server + "|" + suspensionTime.getMillis());
+ }
+
@Override
public boolean equals(Object rhsObject) {
if (!(rhsObject instanceof SuspendingTServer)) {
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/SuspendingTServerTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/SuspendingTServerTest.java
new file mode 100644
index 00..7826915ed5
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/SuspendingTServerTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.util.time.SteadyTime;
+import org.junit.jupiter.api.Test;
+
+import com.google.common.net.HostAndPort;
+
+public class SuspendingTServerTest {
+ @Test
+ public void testToFromValue() {
+SteadyTime suspensionTime = SteadyTime.from(System.currentTimeMillis(),
TimeUnit.MILLISECONDS);
+TServerInstance ser1 = new
TServerInstance(HostAndPort.fromParts("server1", 8555), "s001");
+
+var val1 = SuspendingTServer.toValue(ser1, suspensionTime);
+var st1 = SuspendingTServer.fromValue(val1);
+assertEquals(HostAndPort.fromParts("server1", 8555), st1.server);
+assertEquals(suspensionTime, st1.suspensionTime);
+assertEquals(val1, st1.toValue());
+var st2 = new SuspendingTServer(Host
(accumulo) branch elasticity updated: speeds up tablet mgmt iterator (#4568)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f7e6183271 speeds up tablet mgmt iterator (#4568)
f7e6183271 is described below
commit f7e61832716a196b6b2e9d61666561ef66e27929
Author: Keith Turner
AuthorDate: Sat May 18 13:20:22 2024 -0400
speeds up tablet mgmt iterator (#4568)
Based on profiling tablet servers while running SplitMillionIT, made
the following changes related to TabletManagementIterator
* Made TabletMgmtIterator extend WholeRowIterator instead of stack on
top of it. This avoids encoding->decoding->encoding rows on the
iterator stack.
* Avoided lookuping and parsing multiple table props for each tablet
inorder to determine if it needs split. Moved this be done once per
table.
* Changed a stream to compute file size sum into a loop.
---
.../core/manager/state/TabletManagement.java | 14 +-
.../core/metadata/schema/TabletMetadata.java | 11 +-
.../manager/state/TabletManagementIterator.java| 164 +++--
.../server/manager/state/ZooTabletStateStore.java | 5 +-
.../apache/accumulo/server/split/SplitUtils.java | 8 -
.../state/TabletManagementIteratorTest.java| 6 +-
.../server/manager/state/TabletManagementTest.java | 7 +-
.../manager/ManagerClientServiceHandler.java | 5 +-
8 files changed, 118 insertions(+), 102 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
index 0649791623..7b13e76b7f 100644
---
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
+++
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
@@ -23,6 +23,7 @@ import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
+import java.util.function.BiConsumer;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
@@ -58,17 +59,18 @@ public class TabletManagement {
NEEDS_VOLUME_REPLACEMENT;
}
- public static void addActions(final SortedMap decodedRow,
+ public static void addActions(final BiConsumer bic, final Text
row,
final Set actions) {
-final Key reasonsKey = new Key(decodedRow.firstKey().getRow(),
REASONS_COLUMN_NAME, EMPTY);
+final Key reasonsKey = new Key(row, REASONS_COLUMN_NAME, EMPTY);
final Value reasonsValue = new Value(Joiner.on(',').join(actions));
-decodedRow.put(reasonsKey, reasonsValue);
+bic.accept(reasonsKey, reasonsValue);
}
- public static void addError(final SortedMap decodedRow, final
Exception error) {
-final Key errorKey = new Key(decodedRow.firstKey().getRow(),
ERROR_COLUMN_NAME, EMPTY);
+ public static void addError(final BiConsumer bic, final Text row,
+ final Exception error) {
+final Key errorKey = new Key(row, ERROR_COLUMN_NAME, EMPTY);
final Value errorValue = new Value(error.getMessage());
-decodedRow.put(errorKey, errorValue);
+bic.accept(errorKey, errorValue);
}
private final Set actions;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 6f2ffd6237..ef1fd5ea9c 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -156,8 +156,15 @@ public class TabletMetadata {
this.compacted = tmBuilder.compacted.build();
this.userCompactionsRequested = tmBuilder.userCompactionsRequested.build();
this.unSplittableMetadata = tmBuilder.unSplittableMetadata;
-this.fileSize =
-Suppliers.memoize(() ->
files.values().stream().mapToLong(DataFileValue::getSize).sum());
+this.fileSize = Suppliers.memoize(() -> {
+ // This code was using a java stream. While profiling SplitMillionIT,
the stream was showing
+ // up as hot when scanning 1 million tablets. Converted to a for loop to
improve performance.
+ long sum = 0;
+ for (var dfv : files.values()) {
+sum += dfv.getSize();
+ }
+ return sum;
+});
this.extent =
Suppliers.memoize(() -> new KeyExtent(getTableId(), getEndRow(),
getPrevEndRow()));
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
index b9c64c7f84..e5c7bd32b4 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
+++
b/server/bas
(accumulo) branch elasticity updated: Fixed performance issues that were impacting SplitMillionIT (#4563)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e68f9dd2ef Fixed performance issues that were impacting SplitMillionIT
(#4563)
e68f9dd2ef is described below
commit e68f9dd2ef85677f52b922c4e34ef07bd1e13e15
Author: Keith Turner
AuthorDate: Sat May 18 13:19:16 2024 -0400
Fixed performance issues that were impacting SplitMillionIT (#4563)
The following changes were made as result of running Java Flight
Recorder repeatedly on the manager and tablet server while
SplitMillionIT was running. After these changes the following methods
would not show up as much in the JFR results.
* Sped up validation of FateId. When deleting 1 million tablets, 1
million fate id are written and then read. Was seeing the regex for
validation show up when reading 1 million tablets w/ fate ids.
* Sped up getting children from ZooCache. The code related to service
locks was calling this.
* Sped up parsing of server locks by speeding up the UUID validation,
that is where it was spending most of its time.
* Sped up metadata constraint. Seeing conditional mutation metadata
updates spend a lot of time checking metadata constraints.
* Sped up the conditional check for absent walogs by removing the
creation of an empty TabletMetadata object
* Sped up SetEncodingIterator.encode by having a special case for size
1 and avoiding streams for size >1.
* Increased memory of manager and tsevers in SplitMillionIT because GC
pauses were being seen
* Pre split the metadata table in SplitMillionIT. This allowed the
tablets to spread across the two tablet servers. Pre splitting the
metadata table uncovered a bug. The add splits table operation would
fail if metadata tables it needed were not hosted. Fixed this bug.
* Made some other misc changes for little things that were seen in
profiling.
SplitMillionIT is now running faster, however it still does not seem as
fast as it used to be. Further investigation is needed. These changes
are mostly good general performance fixes. Can follow up wit more fixes
as investigation continues.
Co-authored-by: Dave Marion
---
.../core/clientImpl/TableOperationsImpl.java | 14 +--
.../java/org/apache/accumulo/core/fate/Fate.java | 9 +
.../java/org/apache/accumulo/core/fate/FateId.java | 43 ++--
.../accumulo/core/fate/zookeeper/ZooCache.java | 14 ---
.../accumulo/core/fate/zookeeper/ZooUtil.java | 3 +-
.../org/apache/accumulo/core/lock/ServiceLock.java | 9 ++---
.../org/apache/accumulo/core/util/UuidUtil.java| 46 ++
.../org/apache/accumulo/core/fate/FateIdTest.java | 24 +++
.../server/constraints/MetadataConstraints.java| 10 ++---
.../metadata/ConditionalTabletMutatorImpl.java | 7 +++-
.../metadata/iterators/SetEncodingIterator.java| 22 ---
.../manager/tableOps/merge/DeleteTablets.java | 8 ++--
.../accumulo/test/functional/SplitMillionIT.java | 25 ++--
13 files changed, 188 insertions(+), 46 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 293177a557..c6f33c85b5 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -538,11 +538,19 @@ public class TableOperationsImpl extends
TableOperationsHelper {
var split = iterator.next();
try {
+Retry retry =
Retry.builder().infiniteRetries().retryAfter(Duration.ofMillis(100))
+
.incrementBy(Duration.ofMillis(100)).maxWait(Duration.ofSeconds(2)).backOffFactor(1.5)
+.logInterval(Duration.ofMinutes(3)).createRetry();
+
var tablet = tabLocator.findTablet(context, split, false,
LocationNeed.NOT_REQUIRED);
-if (tablet == null) {
+while (tablet == null) {
context.requireTableExists(tableId, tableName);
- throw new IllegalStateException("Unable to find a tablet for split "
+ split
- + " in table " + tableName + " " + tableId);
+ try {
+retry.waitForNextAttempt(log, "Find tablet in " + tableId + "
containing " + split);
+ } catch (InterruptedException e) {
+throw new RuntimeException(e);
+ }
+ tablet = tabLocator.findTablet(context, split, false,
LocationNeed.NOT_REQUIRED);
}
if (split.equals(tablet.getExtent().endRow())) {
diff --g
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 39c92710683bf75c41fd70e1f21b39ad376dd84d Merge: 4146721be2 220cdc54da Author: Keith Turner AuthorDate: Mon May 13 17:44:27 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch elasticity updated (4146721be2 -> 39c9271068)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4146721be2 Move TestAmple to test module (#4557) add c88d368b2d Fixes race condition that caused unneeded user compactions to run (#4554) add 220cdc54da Merge branch '2.1' new 39c9271068 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 220cdc54da5ba779675d635bcbcc02a6514cd5e6
Merge: 7baa5c76ab c88d368b2d
Author: Keith Turner
AuthorDate: Mon May 13 17:35:55 2024 -0400
Merge branch '2.1'
.../accumulo/tserver/tablet/CompactableImpl.java | 31 +--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 8 +++
.../tablet/CompactableImplFileManagerTest.java | 61 ++
3 files changed, 86 insertions(+), 14 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index d519ffd172,37b23dc6c4..c17f808ab2
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -232,15 -247,32 +232,33 @@@ public class CompactableImpl implement
protected abstract long getNanoTime();
- boolean initiateSelection(CompactionKind kind) {
+ /**
+ * @return the last id of the last successful user compaction
+ */
+ protected abstract long getLastCompactId();
+
+ boolean initiateSelection(CompactionKind kind, Long compactionId) {
- Preconditions
- .checkArgument(kind == DeprecatedCompactionKind.SELECTOR || kind ==
CompactionKind.USER);
+ Preconditions.checkArgument(
- kind == CompactionKind.SELECTOR && compactionId == null
++ kind == DeprecatedCompactionKind.SELECTOR && compactionId == null
+ || kind == CompactionKind.USER && compactionId != null,
+ "Unexpected kind and/or compaction id: %s %s", kind, compactionId);
- if (selectStatus == FileSelectionStatus.NOT_ACTIVE || (kind ==
CompactionKind.USER
- && selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR)
- && selectStatus != FileSelectionStatus.SELECTING)) {
+ if (selectStatus == FileSelectionStatus.NOT_ACTIVE
+ || (kind == CompactionKind.USER && selectKind ==
DeprecatedCompactionKind.SELECTOR
+ && noneRunning(DeprecatedCompactionKind.SELECTOR)
+ && selectStatus != FileSelectionStatus.SELECTING)) {
+
+ // Check compaction id when a lock is held and no other user
compactions have files
+ // selected, at this point the results of any previous user
compactions should be seen. If
+ // user compaction is currently running, then will not get this far
because of the checks a
+ // few lines up.
+ if (kind == CompactionKind.USER && getLastCompactId() >=
compactionId) {
+ // This user compaction has already completed, so no need to
initiate selection of files
+ // for user compaction.
+ return false;
+ }
+
selectStatus = FileSelectionStatus.NEW;
selectKind = kind;
selectedFiles.clear();
diff --cc
server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
index 35d5119dfa,743ac8feca..2f04a875f2
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
@@@ -28,8 -30,10 +29,9 @@@ import static org.junit.jupiter.api.Ass
import java.time.Duration;
import java.util.HashSet;
-import java.util.List;
import java.util.Optional;
import java.util.Set;
+ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@@ -111,10 -117,10 +113,10 @@@ public class CompactableImplFileManager
assertFalse(fileMgr.reserveFiles(staleJob));
assertEquals(newFiles("F0.rf", "F1.rf", "F2.rf", "F3.rf"),
-fileMgr.getCandidates(tabletFiles, SYSTEM, false));
-assertNoCandidates(fileMgr, tabletFiles, CHOP, USER, SELECTOR);
+fileMgr.getCandidates(tabletFiles, SYSTEM));
+assertNoCandidates(fileMgr, tabletFiles, USER, SELECTOR);
- assertTrue(fileMgr.initiateSelection(USER));
+ assertTrue(fileMgr.initiateSelection(USER, 1L));
assertFalse(fileMgr.reserveFiles(staleJob));
@@@ -372,6 -380,96 +374,39 @@@
}
- @Test
- public void testChop() {
-TestFileManager fileMgr = new TestFileManager();
-
-// simulate a compaction because files that were created by compaction
are remembered as not
-// needing a chop
-var job1 = newJob(SYSTEM, "F0.rf", "F1.rf");
-assertTrue(fileMgr.reserveFiles(job1));
-fileMgr.completed(job1, newFile("C5.rf"));
-
-var tabletFiles = newFiles("C5.rf", "
(accumulo) branch main updated (7baa5c76ab -> 220cdc54da)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7baa5c76ab Merge branch '2.1' add c88d368b2d Fixes race condition that caused unneeded user compactions to run (#4554) new 220cdc54da Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/tserver/tablet/CompactableImpl.java | 31 +-- .../org/apache/accumulo/tserver/tablet/Tablet.java | 8 +++ .../tablet/CompactableImplFileManagerTest.java | 61 ++ 3 files changed, 86 insertions(+), 14 deletions(-)
(accumulo) branch 2.1 updated: Fixes race condition that caused unneeded user compactions to run (#4554)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new c88d368b2d Fixes race condition that caused unneeded user compactions
to run (#4554)
c88d368b2d is described below
commit c88d368b2d3eb6384d42739c5322148bf9bcb219
Author: Keith Turner
AuthorDate: Mon May 13 17:04:19 2024 -0400
Fixes race condition that caused unneeded user compactions to run (#4554)
The following is an example of the problem this change fixes
1. Thread 1: A user compaction is currently running for a tablet
2. Thread 2: Tablet server receives a compaction request RPC from manager
and it
checks to see if the compaction is needed for the same tablet. If
finds it is needed.
3. Thread 1: completes user compaction, so a compaction is no longer
needed for the tablet
4. Thread 2: Initiates a user compaction of the tablet because its
check in step 2 passed.
This change adds a second check in step 4 above that should prevent this
race
condition because the check is done at a point when its known no
concurrent user compaction is running. The original check was left as a
fail fast check, but a comment was added explaining its not sufficient
for correctness.
---
.../accumulo/tserver/tablet/CompactableImpl.java | 30 +--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 8 +++
.../tablet/CompactableImplFileManagerTest.java | 63 ++
3 files changed, 88 insertions(+), 13 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index fc84c038aa..37b23dc6c4 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -247,13 +247,32 @@ public class CompactableImpl implements Compactable {
protected abstract long getNanoTime();
-boolean initiateSelection(CompactionKind kind) {
+/**
+ * @return the last id of the last successful user compaction
+ */
+protected abstract long getLastCompactId();
+
+boolean initiateSelection(CompactionKind kind, Long compactionId) {
- Preconditions.checkArgument(kind == CompactionKind.SELECTOR || kind ==
CompactionKind.USER);
+ Preconditions.checkArgument(
+ kind == CompactionKind.SELECTOR && compactionId == null
+ || kind == CompactionKind.USER && compactionId != null,
+ "Unexpected kind and/or compaction id: %s %s", kind, compactionId);
if (selectStatus == FileSelectionStatus.NOT_ACTIVE || (kind ==
CompactionKind.USER
&& selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR)
&& selectStatus != FileSelectionStatus.SELECTING)) {
+
+// Check compaction id when a lock is held and no other user
compactions have files
+// selected, at this point the results of any previous user
compactions should be seen. If
+// user compaction is currently running, then will not get this far
because of the checks a
+// few lines up.
+if (kind == CompactionKind.USER && getLastCompactId() >= compactionId)
{
+ // This user compaction has already completed, so no need to
initiate selection of files
+ // for user compaction.
+ return false;
+}
+
selectStatus = FileSelectionStatus.NEW;
selectKind = kind;
selectedFiles.clear();
@@ -728,6 +747,11 @@ public class CompactableImpl implements Compactable {
protected long getNanoTime() {
return System.nanoTime();
}
+
+ @Override
+ protected long getLastCompactId() {
+return tablet.getLastCompactId();
+ }
};
}
@@ -1030,7 +1054,7 @@ public class CompactableImpl implements Compactable {
return;
}
- if (fileMgr.initiateSelection(kind)) {
+ if (fileMgr.initiateSelection(kind, compactionId)) {
this.chelper = localHelper;
this.compactionId = compactionId;
this.compactionConfig = compactionConfig;
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 6a646c4f57..1fac59303c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -174,6 +174,10 @@ public class Tablet extends TabletBase {
private AtomicLong lastFlushID = new AtomicLong(-1);
private AtomicLong lastComp
(accumulo) branch elasticity updated (0edc48f910 -> 1ca060a77c)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 0edc48f910 Create a test Ample implementation (#4415) add d4e2ae729a reduce the frequency that the list of scan servers is sorted (#4547) add 7baa5c76ab Merge branch '2.1' new 1ca060a77c Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/core/clientImpl/ClientContext.java | 4 ++-- .../apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1ca060a77cee8121af6495238fb9d037b7f96007 Merge: 0edc48f910 7baa5c76ab Author: Keith Turner AuthorDate: Mon May 13 10:52:58 2024 -0400 Merge branch 'main' into elasticity .../main/java/org/apache/accumulo/core/clientImpl/ClientContext.java | 4 ++-- .../apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 7baa5c76abe1dc459d6d554bd5d19cf35c899756 Merge: 861d28b64b d4e2ae729a Author: Keith Turner AuthorDate: Mon May 13 10:48:03 2024 -0400 Merge branch '2.1' .../main/java/org/apache/accumulo/core/clientImpl/ClientContext.java | 4 ++-- .../apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index acc06022e8,e4ba0028c9..093aa7f04e --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@@ -19,8 -19,10 +19,9 @@@ package org.apache.accumulo.core.clientImpl; import static com.google.common.base.Preconditions.checkArgument; + import static com.google.common.base.Suppliers.memoize; import static com.google.common.base.Suppliers.memoizeWithExpiration; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
(accumulo) branch main updated (861d28b64b -> 7baa5c76ab)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 861d28b64b improves suspension IT (#4550) add d4e2ae729a reduce the frequency that the list of scan servers is sorted (#4547) new 7baa5c76ab Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/core/clientImpl/ClientContext.java | 4 ++-- .../apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
(accumulo) branch 2.1 updated: reduce the frequency that the list of scan servers is sorted (#4547)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d4e2ae729a reduce the frequency that the list of scan servers is
sorted (#4547)
d4e2ae729a is described below
commit d4e2ae729a34b419148f2d8bd62c12276f4f7733
Author: Keith Turner
AuthorDate: Mon May 13 10:46:39 2024 -0400
reduce the frequency that the list of scan servers is sorted (#4547)
The provided plugin for working with scan servers sorts the list of scan
servers fairly frequently. This list will not change too often, so it
could be sorted less to offer a better amortized cost as the total
number of scan servers grows.
Also outside of the plugin, the plugin was being reset every 100ms.
Removed this to give the plugin more control. Also there is currently
no way to change client config for a created Accumulo client, so there
is no need to recreate the plugin.
---
.../main/java/org/apache/accumulo/core/clientImpl/ClientContext.java | 4 ++--
.../apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
index d02a2c743e..e4ba0028c9 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.core.clientImpl;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Suppliers.memoize;
import static com.google.common.base.Suppliers.memoizeWithExpiration;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
@@ -232,8 +233,7 @@ public class ClientContext implements AccumuloClient {
saslSupplier = memoizeWithExpiration(
() -> SaslConnectionParams.from(getConfiguration(),
getCredentials().getToken()), 100,
MILLISECONDS);
-scanServerSelectorSupplier =
-memoizeWithExpiration(this::createScanServerSelector, 100,
MILLISECONDS);
+scanServerSelectorSupplier = memoize(this::createScanServerSelector);
this.singletonReservation = Objects.requireNonNull(reservation);
this.tableops = new TableOperationsImpl(this);
this.namespaceops = new NamespaceOperationsImpl(this, tableops);
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
b/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
index 2e792180cc..c77231216f 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
@@ -303,7 +303,7 @@ public class ConfigurableScanServerSelector implements
ScanServerSelector {
.computeIfAbsent(sserver.getGroup(), k -> new
ArrayList<>()).add(sserver.getAddress()));
groupedServers.values().forEach(ssAddrs -> Collections.sort(ssAddrs));
return groupedServers;
-}, 100, TimeUnit.MILLISECONDS);
+}, 3, TimeUnit.SECONDS);
var opts = params.getOptions();
(accumulo) branch elasticity updated (ef34a4890a -> cdaac866da)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from ef34a4890a Merge remote-tracking branch 'upstream/main' into elasticity add 0a0d519912 adds new metrics for cache evictions (#4549) add 861d28b64b improves suspension IT (#4550) new cdaac866da Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/client/rfile/RFileScanner.java | 5 + .../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 5 + .../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 5 + .../java/org/apache/accumulo/core/metrics/MetricsProducer.java| 4 .../main/java/org/apache/accumulo/core/spi/cache/BlockCache.java | 7 +++ .../main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java | 8 .../java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java | 3 +++ 7 files changed, 37 insertions(+)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit cdaac866da85449227864caabc52fc00ee83fe56 Merge: ef34a4890a 861d28b64b Author: Keith Turner AuthorDate: Fri May 10 18:20:33 2024 -0400 Merge remote-tracking branch 'upstream/main' into elasticity .../java/org/apache/accumulo/core/client/rfile/RFileScanner.java | 5 + .../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 5 + .../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 5 + .../java/org/apache/accumulo/core/metrics/MetricsProducer.java| 4 .../main/java/org/apache/accumulo/core/spi/cache/BlockCache.java | 7 +++ .../main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java | 8 .../java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java | 3 +++ 7 files changed, 37 insertions(+)
(accumulo) branch main updated: improves suspension IT (#4550)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 861d28b64b improves suspension IT (#4550)
861d28b64b is described below
commit 861d28b64ba80b82300306df148ea3f969e99e6d
Author: Keith Turner
AuthorDate: Fri May 10 18:18:08 2024 -0400
improves suspension IT (#4550)
---
.../main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
index 8ebb75378b..c32cd927c0 100644
---
a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
@@ -292,6 +292,9 @@ public class SuspendedTabletsIT extends ConfigurableMacBase
{
Thread.sleep(1000);
ds = TabletLocations.retrieve(ctx, tableName);
}
+
+// Ensure all suspension markers in the metadata table were cleared.
+assertTrue(ds.suspended.isEmpty());
} else {
throw new IllegalStateException("Unknown action " + action);
}
(accumulo) branch main updated: adds new metrics for cache evictions (#4549)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 0a0d519912 adds new metrics for cache evictions (#4549)
0a0d519912 is described below
commit 0a0d5199123260afe778ba1c573b0edd02a235ec
Author: Keith Turner
AuthorDate: Fri May 10 18:17:50 2024 -0400
adds new metrics for cache evictions (#4549)
---
.../java/org/apache/accumulo/core/client/rfile/RFileScanner.java | 5 +
.../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 5 +
.../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java | 5 +
.../java/org/apache/accumulo/core/metrics/MetricsProducer.java| 4
.../main/java/org/apache/accumulo/core/spi/cache/BlockCache.java | 7 +++
.../main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java | 8
6 files changed, 34 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
index 79ea0bddfd..b62fc84a3c 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
@@ -141,6 +141,11 @@ class RFileScanner extends ScannerOptions implements
Scanner {
public long requestCount() {
return 0L;
}
+
+@Override
+public long evictionCount() {
+ return 0L;
+}
};
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
index 2167fdcfb1..a373823e20 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
@@ -633,6 +633,11 @@ public class LruBlockCache extends
SynchronousLoadingBlockCache implements Block
return accessCount.get();
}
+@Override
+public long evictionCount() {
+ return getEvictedCount();
+}
+
public long getMissCount() {
return missCount.get();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
index 46a07682bd..ca3a61372f 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
@@ -113,6 +113,11 @@ public final class TinyLfuBlockCache implements BlockCache
{
public long requestCount() {
return stats.requestCount();
}
+
+ @Override
+ public long evictionCount() {
+return stats.evictionCount();
+ }
};
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index abfdfc9b5a..56ef2cdfbf 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@ -655,11 +655,15 @@ public interface MetricsProducer {
String METRICS_BLOCKCACHE_PREFIX = "accumulo.blockcache.";
String METRICS_BLOCKCACHE_INDEX_HITCOUNT = METRICS_BLOCKCACHE_PREFIX +
"index.hitcount";
String METRICS_BLOCKCACHE_INDEX_REQUESTCOUNT = METRICS_BLOCKCACHE_PREFIX +
"index.requestcount";
+ String METRICS_BLOCKCACHE_INDEX_EVICTIONCOUNT = METRICS_BLOCKCACHE_PREFIX +
"index.evictioncount";
String METRICS_BLOCKCACHE_DATA_HITCOUNT = METRICS_BLOCKCACHE_PREFIX +
"data.hitcount";
String METRICS_BLOCKCACHE_DATA_REQUESTCOUNT = METRICS_BLOCKCACHE_PREFIX +
"data.requestcount";
+ String METRICS_BLOCKCACHE_DATA_EVICTIONCOUNT = METRICS_BLOCKCACHE_PREFIX +
"data.evictioncount";
String METRICS_BLOCKCACHE_SUMMARY_HITCOUNT = METRICS_BLOCKCACHE_PREFIX +
"summary.hitcount";
String METRICS_BLOCKCACHE_SUMMARY_REQUESTCOUNT =
METRICS_BLOCKCACHE_PREFIX + "summary.requestcount";
+ String METRICS_BLOCKCACHE_SUMMARY_EVICTIONCOUNT =
+ METRICS_BLOCKCACHE_PREFIX + "summary.evictioncount";
/**
* Build Micrometer Meter objects and register them with the registry
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/cache/BlockCache.java
b/core/src/main/java/org/apache/accumulo/core/spi/cache/BlockCache.java
index d9001490cf..9ebe3fc962 100644
--- a/core/src/main/java/org/apache/accumulo/core/spi/cache/BlockCache.java
+++ b/core/src/main/java/org/apache/accumulo/core/spi
(accumulo) branch main updated: simplifies handling of time in unload handler (#4548)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5aca4f67bc simplifies handling of time in unload handler (#4548)
5aca4f67bc is described below
commit 5aca4f67bc18d221e766a9aec17e6dae0e153ea5
Author: Keith Turner
AuthorDate: Fri May 10 16:53:01 2024 -0400
simplifies handling of time in unload handler (#4548)
---
.../org/apache/accumulo/core/util/time/SteadyTime.java | 5 +
.../apache/accumulo/tserver/TabletClientHandler.java | 5 +++--
.../apache/accumulo/tserver/UnloadTabletHandler.java | 18 --
3 files changed, 16 insertions(+), 12 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
b/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
index 4007b514c0..d16f15c201 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
@@ -56,6 +56,10 @@ public class SteadyTime implements Comparable {
return time.minus(other.getDuration());
}
+ public SteadyTime plus(Duration other) {
+return SteadyTime.from(time.plus(other));
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -90,4 +94,5 @@ public class SteadyTime implements Comparable {
public static SteadyTime from(Duration time) {
return new SteadyTime(time);
}
+
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
index 7a6358b2b0..af16beed4c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
@@ -103,6 +103,7 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.Halt;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.threads.Threads;
+import org.apache.accumulo.core.util.time.SteadyTime;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.compaction.CompactionInfo;
import org.apache.accumulo.server.compaction.FileCompactor;
@@ -1075,8 +1076,8 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
KeyExtent extent = KeyExtent.fromThrift(textent);
-server.resourceManager.addMigration(extent,
-new UnloadTabletHandler(server, extent, goal, requestTime));
+server.resourceManager.addMigration(extent, new
UnloadTabletHandler(server, extent, goal,
+SteadyTime.from(requestTime, TimeUnit.MILLISECONDS)));
}
@Override
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
index 1458901202..27c1048b24 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
@@ -18,11 +18,6 @@
*/
package org.apache.accumulo.tserver;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
-
-import java.util.concurrent.TimeUnit;
-
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.manager.thrift.TabletLoadState;
@@ -31,6 +26,7 @@ import org.apache.accumulo.core.metadata.TabletLocationState;
import
org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.tablet.thrift.TUnloadTabletGoal;
+import org.apache.accumulo.core.util.time.NanoTime;
import org.apache.accumulo.core.util.time.SteadyTime;
import org.apache.accumulo.server.manager.state.DistributedStoreException;
import org.apache.accumulo.server.manager.state.TabletStateStore;
@@ -43,15 +39,17 @@ class UnloadTabletHandler implements Runnable {
private static final Logger log =
LoggerFactory.getLogger(UnloadTabletHandler.class);
private final KeyExtent extent;
private final TUnloadTabletGoal goalState;
- private final long requestTimeSkew;
+ private final SteadyTime requestTime;
+ private final NanoTime createTime;
private final TabletServer server;
public UnloadTabletHandler(TabletServer server, KeyExtent extent,
TUnloadTabletGoal goalState,
- long requestTime) {
+ SteadyTime requestTime) {
this.extent = extent;
this.goalState = goalState;
this.server = server;
-this.requestTimeSkew = requestTime -
NANOSECONDS.toMillis(System.nanoTime());
+this.requestTime = requestTime
(accumulo) branch main updated: Update Ample to use SteadyTime for suspension (#4545)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 368cdb36ae Update Ample to use SteadyTime for suspension (#4545)
368cdb36ae is described below
commit 368cdb36aefdc4ebe3f028feca2e425b418461dd
Author: Christopher L. Shannon
AuthorDate: Fri May 10 14:32:34 2024 -0400
Update Ample to use SteadyTime for suspension (#4545)
SteadyTime was added in #4494 which creates a strong type for tracking
the total duration for the time the cluster has had a manager. This time
is used for suspension so we this updates Ample to use the new type
instead of just a long.
---
.../org/apache/accumulo/core/logging/TabletLogger.java | 7 +++
.../accumulo/core/metadata/SuspendingTServer.java | 17 ++---
.../org/apache/accumulo/core/metadata/schema/Ample.java | 3 ++-
.../org/apache/accumulo/core/util/time/SteadyTime.java | 9 +
.../core/metadata/schema/TabletMetadataTest.java| 9 ++---
.../server/manager/state/LoggingTabletStateStore.java | 7 +++
.../server/manager/state/MetaDataStateStore.java| 11 ++-
.../accumulo/server/manager/state/TabletStateStore.java | 5 +++--
.../server/manager/state/ZooTabletStateStore.java | 3 ++-
.../accumulo/server/metadata/TabletMutatorBase.java | 3 ++-
.../org/apache/accumulo/manager/TabletGroupWatcher.java | 4 ++--
.../apache/accumulo/tserver/UnloadTabletHandler.java| 8 ++--
12 files changed, 54 insertions(+), 32 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
index d01356cfc7..2209e41c27 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
@@ -37,6 +36,7 @@ import
org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.time.SteadyTime;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,10 +75,9 @@ public class TabletLogger {
locLog.debug("Loading {} on {}", extent, server);
}
- public static void suspended(KeyExtent extent, HostAndPort server, long
time, TimeUnit timeUnit,
+ public static void suspended(KeyExtent extent, HostAndPort server,
SteadyTime time,
int numWalogs) {
-locLog.debug("Suspended {} to {} at {} ms with {} walogs", extent, server,
-timeUnit.toMillis(time), numWalogs);
+locLog.debug("Suspended {} to {} at {} ms with {} walogs", extent, server,
time, numWalogs);
}
public static void unsuspended(KeyExtent extent) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
b/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
index 9f59a30cd8..e481369a21 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
@@ -19,8 +19,10 @@
package org.apache.accumulo.core.metadata;
import java.util.Objects;
+import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.time.SteadyTime;
import com.google.common.net.HostAndPort;
@@ -29,21 +31,22 @@ import com.google.common.net.HostAndPort;
*/
public class SuspendingTServer {
public final HostAndPort server;
- public final long suspensionTime;
+ public final SteadyTime suspensionTime;
- SuspendingTServer(HostAndPort server, long suspensionTime) {
+ SuspendingTServer(HostAndPort server, SteadyTime suspensionTime) {
this.server = Objects.requireNonNull(server);
-this.suspensionTime = suspensionTime;
+this.suspensionTime = Objects.requireNonNull(suspensionTime);
}
public static SuspendingTServer fromValue(Value value) {
String valStr = value.toString();
String[] parts = valStr.split("[|]", 2);
-return new SuspendingTServer(HostAndPort.fromString(parts[0]),
Long.parseLong(parts[1]));
+return new SuspendingTServer(HostAndPort.fromString(parts[0]),
+SteadyTime.from(Long.parseLong(parts[1]), TimeUnit.MILLISECONDS));
}
- public static Val
(accumulo) branch 2.1 updated: logs hints used to select a scan server (#4520)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 42e218e903 logs hints used to select a scan server (#4520)
42e218e903 is described below
commit 42e218e903117c110106412ba8252941cc86fb57
Author: Keith Turner
AuthorDate: Sun May 5 21:26:00 2024 -0400
logs hints used to select a scan server (#4520)
---
.../accumulo/core/clientImpl/TabletServerBatchReaderIterator.java | 6 --
.../java/org/apache/accumulo/core/clientImpl/ThriftScanner.java | 8 +---
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
index 963a9f2c4a..8f30ded1c6 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
@@ -652,9 +652,11 @@ public class TabletServerBatchReaderIterator implements
Iterator new
HashMap<>());
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
index 26ddf5abc2..604aece29d 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
@@ -567,13 +567,15 @@ public class ThriftScanner {
delay = actions.getDelay();
scanState.busyTimeout = actions.getBusyTimeout();
log.trace(
- "For tablet {} scan server selector chose scan_server:{}
delay:{} busyTimeout:{}",
- loc.tablet_extent, scanServer, delay, scanState.busyTimeout);
+ "For tablet {} using hints {} scan server selector chose
scan_server:{} delay:{} busyTimeout:{}",
+ loc.tablet_extent, scanState.executionHints, scanServer, delay,
+ scanState.busyTimeout);
} else {
newLoc = loc;
delay = actions.getDelay();
scanState.busyTimeout = Duration.ZERO;
- log.trace("For tablet {} scan server selector chose tablet_server",
loc.tablet_extent);
+ log.trace("For tablet {} using hints {} scan server selector chose
tablet_server",
+ loc.tablet_extent, scanState.executionHints);
}
if (!delay.isZero()) {
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8da73ce467f9311d247ac1eaa8062a5f62853276 Merge: 09adb11491 61ca7d8070 Author: Keith Turner AuthorDate: Wed May 1 22:59:21 2024 + Merge branch '2.1' .../coordinator/CompactionCoordinator.java | 8 +++-- .../coordinator/DeadCompactionDetector.java| 11 +-- .../accumulo/manager/TabletGroupWatcher.java | 35 ++ 3 files changed, 38 insertions(+), 16 deletions(-)
(accumulo) branch main updated (09adb11491 -> 8da73ce467)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 09adb11491 enable metrics log to file for ExternalCompactionMetricsIT (#4502) add 2ce89e0ec4 uses specific logger for compaction coordinator status update (#4489) add acaa6dc46c lowers logging level for first seen dead compaction (#4490) add 61ca7d8070 Handles RPC errors when requesting tablet unload (#4497) new 8da73ce467 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../coordinator/CompactionCoordinator.java | 8 +++-- .../coordinator/DeadCompactionDetector.java| 11 +-- .../accumulo/manager/TabletGroupWatcher.java | 35 ++ 3 files changed, 38 insertions(+), 16 deletions(-)
(accumulo) branch 2.1 updated: Handles RPC errors when requesting tablet unload (#4497)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 61ca7d8070 Handles RPC errors when requesting tablet unload (#4497)
61ca7d8070 is described below
commit 61ca7d8070f2eedabf987f5fb471b475babe7da5
Author: Keith Turner
AuthorDate: Wed May 1 18:55:51 2024 -0400
Handles RPC errors when requesting tablet unload (#4497)
The tablet server group watcher loop will not make progress when it
encounters
an RPC error on a single tablet server. It should continue communicating
with
the tablet servers it can inorder to make progress in its loop that assigns
and unassigns tablets.
---
.../accumulo/manager/TabletGroupWatcher.java | 35 ++
1 file changed, 23 insertions(+), 12 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 8171aa..216526d328 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -344,12 +344,17 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
TServerConnection client =
manager.tserverSet.getConnection(location.getServerInstance());
if (client != null) {
- Manager.log.trace("[{}] Requesting TabletServer {} unload {}
{}", store.name(),
- location.getServerInstance(), tls.extent,
goal.howUnload());
- client.unloadTablet(manager.managerLock, tls.extent,
goal.howUnload(),
- manager.getSteadyTime());
- unloaded++;
- totalUnloaded++;
+ try {
+Manager.log.trace("[{}] Requesting TabletServer {} unload
{} {}", store.name(),
+location.getServerInstance(), tls.extent,
goal.howUnload());
+client.unloadTablet(manager.managerLock, tls.extent,
goal.howUnload(),
+manager.getSteadyTime());
+unloaded++;
+totalUnloaded++;
+ } catch (TException tException) {
+Manager.log.warn("[{}] Failed to request tablet unload {}
{} {}", store.name(),
+location.getServerInstance(), tls.extent,
goal.howUnload(), tException);
+ }
} else {
Manager.log.warn("Could not connect to server {}", location);
}
@@ -1036,13 +1041,19 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
}
tLists.assignments.addAll(tLists.assigned);
for (Assignment a : tLists.assignments) {
- TServerConnection client = manager.tserverSet.getConnection(a.server);
- if (client != null) {
-client.assignTablet(manager.managerLock, a.tablet);
- } else {
-Manager.log.warn("Could not connect to server {}", a.server);
+ try {
+TServerConnection client = manager.tserverSet.getConnection(a.server);
+if (client != null) {
+ client.assignTablet(manager.managerLock, a.tablet);
+ manager.assignedTablet(a.tablet);
+} else {
+ Manager.log.warn("Could not connect to server {} for assignment of
{}", a.server,
+ a.tablet);
+}
+ } catch (TException tException) {
+Manager.log.warn("Could not connect to server {} for assignment of
{}", a.server, a.tablet,
+tException);
}
- manager.assignedTablet(a.tablet);
}
}
(accumulo) branch 2.1 updated: lowers logging level for first seen dead compaction (#4490)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new acaa6dc46c lowers logging level for first seen dead compaction (#4490)
acaa6dc46c is described below
commit acaa6dc46c4fae548889ca20f32cc7cb36b9f406
Author: Keith Turner
AuthorDate: Wed May 1 16:20:43 2024 -0400
lowers logging level for first seen dead compaction (#4490)
For the case of really quick compactions there is a good chance these
finish during the check done by the dead compaction detector. Currently
when this happens a log message is logged about a possible dead compaction
that is a false positive. A large number of quick external compactions
can cause a lot of these false positives.
This change adjust the first time a possible dead compaction is logged
to trace.
---
.../apache/accumulo/coordinator/DeadCompactionDetector.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
index ba4a575ddf..b58f06a31e 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
@@ -117,8 +117,15 @@ public class DeadCompactionDetector {
});
tabletCompactions.forEach((ecid, extent) -> {
- log.debug("Possible dead compaction detected {} {}", ecid, extent);
- this.deadCompactions.merge(ecid, 1L, Long::sum);
+ var count = this.deadCompactions.merge(ecid, 1L, Long::sum);
+ if (count == 1) {
+// The first time a possible dead compaction is seen, for quick
compactions there is a good
+// chance that it is already complete instead of dead. In order to
avoid spamming the logs
+// w/ false positives, log the first seen at trace.
+log.trace("Possible dead compaction detected {} {} {}", ecid, extent,
count);
+ } else {
+log.debug("Possible dead compaction detected {} {} {}", ecid, extent,
count);
+ }
});
// Everything left in tabletCompactions is no longer running anywhere and
should be failed.
(accumulo) branch 2.1 updated: uses specific logger for compaction coordinator status update (#4489)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 2ce89e0ec4 uses specific logger for compaction coordinator status
update (#4489)
2ce89e0ec4 is described below
commit 2ce89e0ec4a6cc57b139e1f33f4a8f51ebb6b211
Author: Keith Turner
AuthorDate: Wed May 1 16:19:55 2024 -0400
uses specific logger for compaction coordinator status update (#4489)
---
.../org/apache/accumulo/coordinator/CompactionCoordinator.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 2b0dee53b7..9ac4ddd27a 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -100,6 +100,9 @@ public class CompactionCoordinator extends AbstractServer
implements CompactionCoordinatorService.Iface, LiveTServerSet.Listener {
private static final Logger LOG =
LoggerFactory.getLogger(CompactionCoordinator.class);
+
+ private static final Logger STATUS_LOG =
+ LoggerFactory.getLogger(CompactionCoordinator.class.getName() +
".compaction.status");
private static final long TIME_BETWEEN_GC_CHECKS = 5000;
protected static final QueueSummaries QUEUE_SUMMARIES = new QueueSummaries();
@@ -585,8 +588,8 @@ public class CompactionCoordinator extends AbstractServer
throw new AccumuloSecurityException(credentials.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED).asThriftException();
}
-LOG.debug("Compaction status update, id: {}, timestamp: {}, update: {}",
externalCompactionId,
-timestamp, update);
+STATUS_LOG.debug("Compaction status update, id: {}, timestamp: {}, update:
{}",
+externalCompactionId, timestamp, update);
final RunningCompaction rc =
RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId));
if (null != rc) {
rc.addUpdate(timestamp, update);
(accumulo) branch 2.1 updated (4839d3bf22 -> 714fdb7e1a)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4839d3bf22 logs when closing tablets are waiting on compactions (#4472) add 714fdb7e1a logs when tablet decides there is nothing to do for user compaction (#4473) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/tserver/tablet/CompactableImpl.java | 9 + 1 file changed, 9 insertions(+)
(accumulo) branch 2.1 updated (8521cb3178 -> 4839d3bf22)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8521cb3178 improves efficiency of migration set clean up (#4474) add 4839d3bf22 logs when closing tablets are waiting on compactions (#4472) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/tserver/tablet/CompactableImpl.java | 13 +++-- 1 file changed, 11 insertions(+), 2 deletions(-)
(accumulo) branch 2.1 updated (7c6572caf3 -> 8521cb3178)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7c6572caf3 fixes bug with starting multiple group of scan servers (#4445) add 8521cb3178 improves efficiency of migration set clean up (#4474) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/manager/Manager.java | 14 +- 1 file changed, 9 insertions(+), 5 deletions(-)
(accumulo) branch elasticity updated: batches reading tablet metadata for refresh (#4439)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new cb25f26ab3 batches reading tablet metadata for refresh (#4439)
cb25f26ab3 is described below
commit cb25f26ab3be77dbfce54e9d1e7234e5974510ec
Author: Keith Turner
AuthorDate: Wed Apr 17 17:38:37 2024 -0400
batches reading tablet metadata for refresh (#4439)
Changes tablet server code that refreshes tablet metadata to read tablet
metadata for multiple tablets in a batch. Also noticed the refresh code
was not handling some race condition correctly and could result in
spurious error messages. It was also not handling closed tablets.
Refactored the code to return false when refresh is not possble for
acceptable reasons instead of throw an error. This will cause the the
RPC to retry for those tablets later. Also remove some code that was
attempting to handle complex race conditions that would be hard to test,
realized if these conditions do happen that it would be best to retry
rather than try to handle them.
Co-authored-by: Dave Marion
---
.../accumulo/tserver/TabletClientHandler.java | 86 -
.../org/apache/accumulo/tserver/tablet/Tablet.java | 137 +
2 files changed, 138 insertions(+), 85 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
index f8532de0b8..a1770b90cb 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -1096,80 +1097,47 @@ public class TabletClientHandler implements
TabletServerClientService.Iface,
// handle that more expensive case if needed.
var tabletsSnapshot = server.getOnlineTablets();
-Set notFound = new HashSet<>();
+Map refreshSessions = new HashMap<>();
+
+// Created this as synchronized list because it's passed to a lambda that
could possibly run in
+// another thread.
+List unableToRefresh = Collections.synchronizedList(new
ArrayList<>());
for (var tkextent : refreshes) {
var extent = KeyExtent.fromThrift(tkextent);
-
var tablet = tabletsSnapshot.get(extent);
if (tablet != null) {
-// ELASTICITY_TODO use a batch reader to read all tablets metadata at
once instead of one by
-// one. This may be a bit tricky from a synchronization perspective
(with multiple tablets
-// and multiple concurrent refresh request), so defer doing this until
after removing
-// functionality from the tablet. No need to make the change now and
have to change it
-// later.
-tablet.refreshMetadata(RefreshPurpose.REFRESH_RPC);
+refreshSessions.put(extent, tablet.startRefresh());
} else {
-notFound.add(extent);
+unableToRefresh.add(extent.toThrift());
}
}
-if (!notFound.isEmpty()) {
- // Some tablets were not found, lets see if they are loading or moved to
online while doing
- // the refreshes above.
- List unableToRefresh = new ArrayList<>();
- List foundTablets = new ArrayList<>();
-
- synchronized (server.unopenedTablets) {
-synchronized (server.openingTablets) {
- synchronized (server.onlineTablets) {
-// Get the snapshot again, however this time nothing will be
changing while we iterate
-// over the snapshot because all three locks are held.
-tabletsSnapshot = server.getOnlineTablets();
-for (var extent : notFound) {
- // TODO investigate if its safe to ignore tablets in the
unopened set because they
- // have not yet read any metadata
- if (server.unopenedTablets.contains(extent)
- || server.openingTablets.contains(extent)) {
-// Can not refresh these tablets that are in the process of
loading, but they may
-// still need refreshing because we don't know when they read
their metadata
-// relative to the refresh event.
-unableToRefresh.add(extent.toThrift());
- } else {
-var tablet = tabletsSnapshot.get(extent);
-if (tablet != null) {
- // Intentionally not calling refresh on the tablet while
holding these locks.
-
(accumulo) branch elasticity updated (016dda7ffe -> d3264f1ffa)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 016dda7ffe removes selector compactions (#4397) add 5bea0f6b26 Deprecates per table selector compactions (#4398) new e3742b0d77 Merge branch 'main' into elasticity new d3264f1ffa removes uneeded deprecation suppression The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/02: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit e3742b0d774d0214dbcd276fa4ffcffe7db4ed4b
Merge: 016dda7ffe 5bea0f6b26
Author: Keith Turner
AuthorDate: Wed Apr 17 16:18:58 2024 -0400
Merge branch 'main' into elasticity
.../java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java | 1 +
1 file changed, 1 insertion(+)
diff --cc
test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
index 67c01286a9,ec9b22f25c..f8d7c31467
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
@@@ -391,10 -323,17 +391,11 @@@ public class CompactionExecutorIT exten
}
@Test
+ @SuppressWarnings("deprecation")
public void testTooManyDeletes() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- Map props = Map.of(Property.TABLE_COMPACTION_SELECTOR.getKey(),
- TooManyDeletesSelector.class.getName(),
- Property.TABLE_COMPACTION_SELECTOR_OPTS.getKey() + "threshold",
".4");
var deleteSummarizerCfg =
SummarizerConfiguration.builder(DeletesSummarizer.class.getName()).build();
- client.tableOperations().create("tmd_selector", new
NewTableConfiguration()
- .setProperties(props).enableSummarization(deleteSummarizerCfg));
client.tableOperations().create("tmd_control1",
new
NewTableConfiguration().enableSummarization(deleteSummarizerCfg));
client.tableOperations().create("tmd_control2",
(accumulo) 02/02: removes uneeded deprecation suppression
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit d3264f1ffa2b6bedaf45f6c5af5b9f7eac439139
Author: Keith Turner
AuthorDate: Wed Apr 17 16:27:06 2024 -0400
removes uneeded deprecation suppression
---
.../java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
index f8d7c31467..67c01286a9 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
@@ -391,7 +391,6 @@ public class CompactionExecutorIT extends
SharedMiniClusterBase {
}
@Test
- @SuppressWarnings("deprecation")
public void testTooManyDeletes() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
var deleteSummarizerCfg =
(accumulo) branch elasticity updated: removes selector compactions (#4397)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 016dda7ffe removes selector compactions (#4397)
016dda7ffe is described below
commit 016dda7ffe6f67a096af38a9e6f23133747c7c2c
Author: Keith Turner
AuthorDate: Wed Apr 17 16:02:00 2024 -0400
removes selector compactions (#4397)
Co-authored-by: Daniel Roberts
---
.../core/client/admin/compaction/CompactionSelector.java | 3 +--
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 9 +
.../apache/accumulo/core/spi/compaction/CompactionKind.java| 6 --
.../core/spi/compaction/RatioBasedCompactionPlanner.java | 10 --
.../accumulo/core/tabletserver/thrift/TCompactionKind.java | 3 ---
core/src/main/thrift/tabletserver.thrift | 6 +++---
.../core/util/compaction/CompactionPrioritizerTest.java| 4 ++--
.../src/main/java/org/apache/accumulo/compactor/ExtCEnv.java | 1 -
.../manager/compaction/coordinator/CompactionCoordinator.java | 2 --
.../compaction/coordinator/commit/CommitCompaction.java| 4 ++--
10 files changed, 13 insertions(+), 35 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
index 281372d43d..311ffc730e 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
@@ -35,8 +35,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
/**
- * This class selects which files a user compaction will compact. It can also
be configured per
- * table to periodically select files to compact.
+ * This class selects which files a user compaction will compact.
*
* @since 2.1.0
*/
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index e33eed2748..b05e0aba4e 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -855,12 +855,6 @@ public enum Property {
+ "specified time. If a system compaction cancels a hold and runs,
then the user compaction"
+ " can reselect and hold files after the system compaction runs.",
"2.1.0"),
- TABLE_COMPACTION_SELECTOR("table.compaction.selector", "",
PropertyType.CLASSNAME,
- "A configurable selector for a table that can periodically select file
for mandatory "
- + "compaction, even if the files do not meet the compaction ratio.",
- "2.1.0"),
- TABLE_COMPACTION_SELECTOR_OPTS("table.compaction.selector.opts.", null,
PropertyType.PREFIX,
- "Options for the table compaction dispatcher.", "2.1.0"),
TABLE_COMPACTION_CONFIGURER("table.compaction.configurer", "",
PropertyType.CLASSNAME,
"A plugin that can dynamically configure compaction output files based
on input files.",
"2.1.0"),
@@ -1414,8 +1408,7 @@ public enum Property {
|| key.startsWith(TABLE_SUMMARIZER_PREFIX.getKey())
|| key.startsWith(TABLE_SCAN_DISPATCHER_OPTS.getKey())
|| key.startsWith(TABLE_COMPACTION_DISPATCHER_OPTS.getKey())
-|| key.startsWith(TABLE_COMPACTION_CONFIGURER_OPTS.getKey())
-|| key.startsWith(TABLE_COMPACTION_SELECTOR_OPTS.getKey()))
+|| key.startsWith(TABLE_COMPACTION_CONFIGURER_OPTS.getKey()))
|| key.startsWith(TABLE_CRYPTO_PREFIX.getKey()));
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionKind.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionKind.java
index 97e64216fc..7667dcf577 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionKind.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionKind.java
@@ -18,8 +18,6 @@
*/
package org.apache.accumulo.core.spi.compaction;
-import org.apache.accumulo.core.client.admin.compaction.CompactionSelector;
-
/**
* @since 2.1.0
* @see org.apache.accumulo.core.spi.compaction
@@ -29,10 +27,6 @@ public enum CompactionKind {
* A system initiated routine compaction.
*/
SYSTEM,
- /**
- * Set of files selected by a {@link CompactionSelector} configured for a
table.
- */
- SELECTOR,
/**
* A user initiated a one time compaction using an Accumulo client.
*/
diff --
(accumulo) branch main updated: Deprecates per table selector compactions (#4398)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5bea0f6b26 Deprecates per table selector compactions (#4398)
5bea0f6b26 is described below
commit 5bea0f6b2674188b5c242e78e177d1a8c11ba4b7
Author: Keith Turner
AuthorDate: Wed Apr 17 15:50:48 2024 -0400
Deprecates per table selector compactions (#4398)
See #4081 for details
---
.../admin/compaction/CompactionSelector.java | 3 +-
.../org/apache/accumulo/core/conf/Property.java| 5 ++-
.../schema/ExternalCompactionMetadata.java | 3 +-
.../core/spi/compaction/CompactionKind.java| 10 ++
.../spi/compaction/DefaultCompactionPlanner.java | 6 ++--
.../util/compaction/CompactionJobPrioritizer.java | 1 +
.../compaction/DeprecatedCompactionKind.java} | 25 -
.../util/compaction/CompactionPrioritizerTest.java | 4 +--
.../accumulo/tserver/tablet/CompactableImpl.java | 41 +-
.../accumulo/tserver/tablet/CompactableUtils.java | 17 +
.../apache/accumulo/tserver/tablet/MajCEnv.java| 1 +
.../tablet/CompactableImplFileManagerTest.java | 2 +-
.../tserver/tablet/CompactableImplTest.java| 9 ++---
.../test/compaction/CompactionExecutorIT.java | 1 +
14 files changed, 75 insertions(+), 53 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
index 281372d43d..713a451dc0 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
@@ -36,7 +36,8 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
/**
* This class selects which files a user compaction will compact. It can also
be configured per
- * table to periodically select files to compact.
+ * table to periodically select files to compact, although per table
functionality is deprecated.
+ * See {@link org.apache.accumulo.core.spi.compaction.CompactionKind#SELECTOR}
for details.
*
* @since 2.1.0
*/
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index f86e4cad3b..7d48c05ae1 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -916,10 +916,13 @@ public enum Property {
+ "specified time. If a system compaction cancels a hold and runs,
then the user compaction"
+ " can reselect and hold files after the system compaction runs.",
"2.1.0"),
+ @Deprecated(since = "3.1")
TABLE_COMPACTION_SELECTOR("table.compaction.selector", "",
PropertyType.CLASSNAME,
"A configurable selector for a table that can periodically select file
for mandatory "
- + "compaction, even if the files do not meet the compaction ratio.",
+ + "compaction, even if the files do not meet the compaction ratio.
This option was deprecated in "
+ + "3.1, see the CompactionKind.SELECTOR enum javadoc for details.",
"2.1.0"),
+ @Deprecated(since = "3.1")
TABLE_COMPACTION_SELECTOR_OPTS("table.compaction.selector.opts.", null,
PropertyType.PREFIX,
"Options for the table compaction dispatcher.", "2.1.0"),
TABLE_COMPACTION_CONFIGURER("table.compaction.configurer", "",
PropertyType.CLASSNAME,
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index a5beddd7ad..4c7e4298ff 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
+import org.apache.accumulo.core.util.compaction.DeprecatedCompactionKind;
public class ExternalCompactionMetadata {
@@ -50,7 +51,7 @@ public class ExternalCompactionMetadata {
CompactionExecutorId ceid, boolean propagateDeletes, boolean
initiallySelectedAll,
Long compactionId) {
if (!initiallySelectedAll && !propagateDeletes
-
(accumulo) branch elasticity updated: fixes continue scan in tablet mgmt iterator (#4457)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 6e0f2a14fa fixes continue scan in tablet mgmt iterator (#4457)
6e0f2a14fa is described below
commit 6e0f2a14fa051adb2c7873877bc59e1bcb0ad0b8
Author: Keith Turner
AuthorDate: Mon Apr 15 11:25:48 2024 -0400
fixes continue scan in tablet mgmt iterator (#4457)
Many places in the accumulo code will read a batch of key/values and
then use the last key in the batch to construct a range to get the next
batch. The last key in the batch will be a non inclusive start key for
the range. The tablet mgmt iterator was not handling this case and
returning the key that should have been excluded.
---
.../manager/state/TabletManagementIterator.java| 6 ++-
.../functional/TabletManagementIteratorIT.java | 51 +-
2 files changed, 44 insertions(+), 13 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
index b3ebe61c1d..3f5397a7a4 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
@@ -217,7 +217,11 @@ public class TabletManagementIterator extends
SkippingIterator {
// can pull this K,V pair from the results by looking at the colf.
TabletManagement.addActions(decodedRow, actions);
}
-topKey = decodedRow.firstKey();
+
+// This key is being created exactly the same way as the whole row
iterator creates keys.
+// This is important for ensuring that seek works as expected in the
continue case. See
+// WholeRowIterator seek function for details, it looks for keys w/o
columns.
+topKey = new Key(decodedRow.firstKey().getRow());
topValue = WholeRowIterator.encodeRow(new
ArrayList<>(decodedRow.keySet()),
new ArrayList<>(decodedRow.values()));
LOG.trace("Returning extent {} with reasons: {}", tm.getExtent(),
actions);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
index 6d74d0c16e..7af3e0ba00 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
@@ -18,8 +18,13 @@
*/
package org.apache.accumulo.test.functional;
+import static
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction.BAD_STATE;
import static
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction.NEEDS_LOCATION_UPDATE;
+import static
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction.NEEDS_RECOVERY;
+import static
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction.NEEDS_SPLITTING;
+import static
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction.NEEDS_VOLUME_REPLACEMENT;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.time.Duration;
@@ -187,6 +192,10 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
assertEquals(expected, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
"Should have four tablets with hosting availability changes");
+ // test continue scan functionality, this test needs a table and tablet
mgmt params that will
+ // return more than one tablet
+ testContinueScan(client, metaCopy1, tabletMgmtParams);
+
// test the assigned case (no location)
removeLocation(client, metaCopy1, t3);
expected =
@@ -210,18 +219,15 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
// Test the recovery cases
createLogEntry(client, metaCopy5, t1);
setTabletAvailability(client, metaCopy5, t1,
TabletAvailability.UNHOSTED.name());
- expected = Map.of(endR1,
- Set.of(NEEDS_LOCATION_UPDATE,
TabletManagement.ManagementAction.NEEDS_RECOVERY));
+ expected = Map.of(endR1, Set.of(NEEDS_LOCATION_UPDATE, NEEDS_RECOVERY));
assertEquals(expected, findTabletsNeedingAttention(client, metaCopy5,
tabletMgmtParams),
"Only 1 of 2 tablets in table t1 should be returned");
setTabletAvailability(client, metaCopy5, t1,
TabletAvailability.ONDEMAND.name());
- expected = Map.of(endR
(accumulo) branch elasticity updated: refactored findTabletsNeedingAttention test to handle new return type. (#4418)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 370a7ded78 refactored findTabletsNeedingAttention test to handle new
return type. (#4418)
370a7ded78 is described below
commit 370a7ded78379ad312e3dc486f671d0c053792d0
Author: Arbaaz Khan
AuthorDate: Fri Apr 12 17:20:47 2024 -0400
refactored findTabletsNeedingAttention test to handle new return type.
(#4418)
---
.../functional/TabletManagementIteratorIT.java | 93 --
1 file changed, 68 insertions(+), 25 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
index 22ae8c6535..6d74d0c16e 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
@@ -18,12 +18,14 @@
*/
package org.apache.accumulo.test.functional;
+import static
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction.NEEDS_LOCATION_UPDATE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -141,15 +143,30 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
// examine a clone of the metadata table, so we can manipulate it
copyTable(client, AccumuloTable.METADATA.tableName(), metaCopy1);
+ var tableId1 = getServerContext().getTableId(t1);
+ var tableId3 = getServerContext().getTableId(t3);
+ var tableId4 = getServerContext().getTableId(t4);
+
+ // Create expected KeyExtents to test output of
findTabletsNeedingAttention
+ KeyExtent endR1 = new KeyExtent(tableId1, new Text("some split"), null);
+ KeyExtent endR3 = new KeyExtent(tableId3, new Text("some split"), null);
+ KeyExtent endR4 = new KeyExtent(tableId4, new Text("some split"), null);
+ KeyExtent prevR1 = new KeyExtent(tableId1, null, new Text("some split"));
+ KeyExtent prevR3 = new KeyExtent(tableId3, null, new Text("some split"));
+ KeyExtent prevR4 = new KeyExtent(tableId4, null, new Text("some split"));
+ Map> expected;
+
TabletManagementParameters tabletMgmtParams = createParameters(client);
- int tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams);
- while (tabletsInFlux > 0) {
+ Map> tabletsInFlux =
+ findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams);
+ while (!tabletsInFlux.isEmpty()) {
log.debug("Waiting for {} tablets for {}", tabletsInFlux, metaCopy1);
UtilWaitThread.sleep(500);
copyTable(client, AccumuloTable.METADATA.tableName(), metaCopy1);
tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams);
}
- assertEquals(0, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
+ expected = Map.of();
+ assertEquals(expected, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
"No tables should need attention");
// The metadata table stabilized and metaCopy1 contains a copy suitable
for testing. Before
@@ -165,72 +182,93 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
// t3 is hosted, setting to never will generate a change to unhost
tablets
setTabletAvailability(client, metaCopy1, t3,
TabletAvailability.UNHOSTED.name());
tabletMgmtParams = createParameters(client);
- assertEquals(4, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
+ expected = Map.of(endR1, Set.of(NEEDS_LOCATION_UPDATE), prevR1,
Set.of(NEEDS_LOCATION_UPDATE),
+ endR3, Set.of(NEEDS_LOCATION_UPDATE), prevR3,
Set.of(NEEDS_LOCATION_UPDATE));
+ assertEquals(expected, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
"Should have four tablets with hosting availability changes");
// test the assigned case (no location)
removeLocation(client, metaCopy1, t3);
- assertEquals(2, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
+ expected =
+ Map.of(endR1, Set.of(NEEDS_LOCATION_UPDATE), prevR1,
Set.of(NEEDS_LOCATION_UPDATE));
+ assertEquals(expected, findTabletsNeedingAttention(client, metaCopy1,
tabletMgmtParams),
"Should have two tablets without a loc");
// Test setting the operation i
(accumulo) branch elasticity updated: removes TODO that is done
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 4081f460ae removes TODO that is done
4081f460ae is described below
commit 4081f460aea41342904929b0317edd5b650b69fb
Author: Keith Turner
AuthorDate: Thu Apr 11 20:12:29 2024 -0400
removes TODO that is done
---
.../base/src/main/java/org/apache/accumulo/server/split/SplitUtils.java | 2 --
1 file changed, 2 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/split/SplitUtils.java
b/server/base/src/main/java/org/apache/accumulo/server/split/SplitUtils.java
index eb5ceec1bf..8f64c461bc 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/split/SplitUtils.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/split/SplitUtils.java
@@ -203,8 +203,6 @@ public class SplitUtils {
var threshold = tableConf.getAsBytes(Property.TABLE_SPLIT_THRESHOLD);
var maxEndRowSize = tableConf.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE);
-// ELASTICITY_TODO rename and deprecate property. This is not executing in
the tablet server
-// anymore.
int maxFilesToOpen = tableConf.getCount(Property.SPLIT_MAXOPEN);
var estimatedSize = tabletMetadata.getFileSize();
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f337357a527b1d8beab4ad6573fdde3229bd101d
Merge: 9fd7338ba4 859694aba8
Author: Keith Turner
AuthorDate: Thu Apr 11 20:07:16 2024 -0400
Merge branch 'main' into elasticity
assemble/bin/accumulo-cluster| 2 +-
.../java/org/apache/accumulo/core/conf/Property.java | 16
.../server/manager/state/TabletManagementIterator.java | 4 ++--
.../org/apache/accumulo/server/split/SplitUtils.java | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
diff --cc assemble/bin/accumulo-cluster
index 82e53dbd4a,f525093404..b157f04c1e
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@@ -156,9 -131,9 +156,9 @@@ function control_service()
for ((inst_id = 1; inst_id <= last_instance_id; inst_id++)); do
ACCUMULO_SERVICE_INSTANCE=""
-[[ $service == "tserver" && ${NUM_TSERVERS:-1} -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE=${inst_id}
-[[ $service == "compactor" ]] &&
ACCUMULO_SERVICE_INSTANCE="${inst_id}_${5}"
-[[ $service == "sserver" ]] && ACCUMULO_SERVICE_INSTANCE="${inst_id}_${5}"
+[[ $service == "tserver" && ${NUM_TSERVERS:-1} -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
+[[ $service == "compactor" ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
- [[ $service == "sserver" && ${NUM_SSERVERS:-1} -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
++[[ $service == "sserver" ]] &&
ACCUMULO_SERVICE_INSTANCE="_${group}_${inst_id}"
if [[ $host == localhost || $host == "$(hostname -s)" || $host ==
"$(hostname -f)" || "$(hostname -I)" =~ $host ]]; then
#
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index d5a9504f28,f86e4cad3b..e33eed2748
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -433,13 -395,15 +433,21 @@@ public enum Property
+ "indefinitely. Default is 0 to block indefinitely. Only valid
when tserver available "
+ "threshold is set greater than 0.",
"1.10.0"),
+ MANAGER_SPLIT_WORKER_THREADS("manager.split.inspection.threadpool.size",
"8", PropertyType.COUNT,
+ "The number of threads used to inspect tablets files to find split
points.", "4.0.0"),
+
+
MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE("manager.compaction.major.service.queue.size",
+ // ELASTICITY_TODO: It might be good to note that there is a priority
queue per compactor
+ // resource group
+ "1", PropertyType.COUNT, "The max size of the priority queue.",
"4.0"),
+ SPLIT_PREFIX("split.", null, PropertyType.PREFIX,
+ "System wide properties related to splitting tablets.", "3.1.0"),
+ SPLIT_MAXOPEN("split.files.max", "300", PropertyType.COUNT,
+ "To find a tablets split points, all RFiles are opened and their
indexes"
+ + " are read. This setting determines how many RFiles can be opened
at once."
- + " When there are more RFiles than this setting multiple passes
must be"
- + " made, which is slower. However opening too many RFiles at once
can cause"
- + " problems.",
++ + " When there are more RFiles than this setting the tablet will be
marked"
++ + " as un-splittable.",
+ "3.1.0"),
// properties that are specific to scan server behavior
@Experimental
SSERV_PREFIX("sserver.", null, PropertyType.PREFIX,
@@@ -545,14 -505,16 +553,6 @@@
TSERV_TOTAL_MUTATION_QUEUE_MAX("tserver.total.mutation.queue.max", "5%",
PropertyType.MEMORY,
"The amount of memory used to store write-ahead-log mutations before
flushing them.",
"1.7.0"),
- @ReplacedBy(property = SPLIT_MAXOPEN)
- @Deprecated(since = "3.1")
--
TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN("tserver.tablet.split.midpoint.files.max",
"300",
-- PropertyType.COUNT,
-- "To find a tablets split points, all RFiles are opened and their
indexes"
-- + " are read. This setting determines how many RFiles can be opened
at once."
-- + " When there are more RFiles than this setting multiple passes
must be"
-- + " made,
(accumulo) branch elasticity updated (9fd7338ba4 -> f337357a52)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9fd7338ba4 Documents causes of error in ConditionalTabletsMutatorImpl (#4449) add 61e81c43d8 falls back to deprecated compaction prop (#4451) add c39599d75d Renames split property to avoid tserver (#4450) add 3885a25df6 corrects illegal json in javadoc (#4446) add 7c6572caf3 fixes bug with starting multiple group of scan servers (#4445) add 859694aba8 Merge branch '2.1' new f337357a52 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/bin/accumulo-cluster| 2 +- .../java/org/apache/accumulo/core/conf/Property.java | 16 .../server/manager/state/TabletManagementIterator.java | 4 ++-- .../org/apache/accumulo/server/split/SplitUtils.java | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 859694aba88b64ce3f9c5559c0f772f987dbb0e4 Merge: c39599d75d 7c6572caf3 Author: Keith Turner AuthorDate: Thu Apr 11 19:22:12 2024 -0400 Merge branch '2.1' assemble/bin/accumulo-cluster | 2 +- .../apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
(accumulo) branch main updated (c39599d75d -> 859694aba8)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from c39599d75d Renames split property to avoid tserver (#4450) add 3885a25df6 corrects illegal json in javadoc (#4446) add 7c6572caf3 fixes bug with starting multiple group of scan servers (#4445) new 859694aba8 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/bin/accumulo-cluster | 2 +- .../apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
(accumulo) branch 2.1 updated: fixes bug with starting multiple group of scan servers (#4445)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 7c6572caf3 fixes bug with starting multiple group of scan servers
(#4445)
7c6572caf3 is described below
commit 7c6572caf336cd76bb794befe7a31dce3503cd4b
Author: Keith Turner
AuthorDate: Thu Apr 11 19:19:36 2024 -0400
fixes bug with starting multiple group of scan servers (#4445)
When trying to start and stop multiple groups of scan servers using the
accumulo-cluster script the groups would interfere with each other.
This change fixes that by making the the scan servers use the group in
the instance id like compactors do.
Ran into this while workiing on # and pulled it out as a stand alone
fix.
---
assemble/bin/accumulo-cluster | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster
index 5dd9de7e4e..d31d65989c 100755
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@ -141,7 +141,7 @@ function control_service() {
ACCUMULO_SERVICE_INSTANCE=""
[[ $service == "tserver" && ${NUM_TSERVERS:-1} -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE=${inst_id}
[[ $service == "compactor" ]] &&
ACCUMULO_SERVICE_INSTANCE="${inst_id}_${5}"
-[[ $service == "sserver" && ${NUM_SSERVERS:-1} -gt 1 ]] &&
ACCUMULO_SERVICE_INSTANCE=${inst_id}
+[[ $service == "sserver" ]] && ACCUMULO_SERVICE_INSTANCE="${inst_id}_${5}"
if [[ $host == localhost || $host == "$(hostname -s)" || $host ==
"$(hostname -f)" || "$(hostname -I)" =~ $host ]]; then
#
(accumulo) branch elasticity updated: Documents causes of error in ConditionalTabletsMutatorImpl (#4449)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 9fd7338ba4 Documents causes of error in ConditionalTabletsMutatorImpl
(#4449)
9fd7338ba4 is described below
commit 9fd7338ba4381cd21bc471bc0bf8d37c04a9b0b3
Author: Keith Turner
AuthorDate: Thu Apr 11 19:18:27 2024 -0400
Documents causes of error in ConditionalTabletsMutatorImpl (#4449)
---
.../accumulo/server/metadata/ConditionalTabletsMutatorImpl.java | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java
index 4401654313..2e999033fa 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java
@@ -273,9 +273,6 @@ public class ConditionalTabletsMutatorImpl implements
Ample.ConditionalTabletsMu
private void
ensureAllExtentsSeen(HashMap resultsMap,
Set extentsSet) {
if (!resultsMap.keySet().equals(Set.copyOf(extents.values( {
- // ELASTICITY_TODO this check can trigger if someone forgets to submit,
could check for
- // that
-
Sets.difference(resultsMap.keySet(), extentsSet)
.forEach(extent -> log.error("Unexpected extent seen in in result
{}", extent));
@@ -286,7 +283,11 @@ public class ConditionalTabletsMutatorImpl implements
Ample.ConditionalTabletsMu
log.error("result seen {} {}", keyExtent, new
Text(result.getMutation().getRow()));
});
- throw new AssertionError("Not all extents were seen, this is
unexpected");
+ // There are at least two possible causes for this condition. First
there is a bug in the
+ // ConditionalWriter, and it did not return a result for a mutation that
was given to it.
+ // Second, Ample code was not used correctly and mutating an extent was
started but never
+ // submitted.
+ throw new IllegalStateException("Not all extents were seen, this is
unexpected.");
}
}
(accumulo) branch 2.1 updated: corrects illegal json in javadoc (#4446)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 3885a25df6 corrects illegal json in javadoc (#4446)
3885a25df6 is described below
commit 3885a25df6f448846fa33f9f337336e49585f73e
Author: Keith Turner
AuthorDate: Thu Apr 11 19:18:11 2024 -0400
corrects illegal json in javadoc (#4446)
---
.../apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index 5a30556f63..73952b7900 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -99,7 +99,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* [
* {"name":"small", "type": "internal", "maxSize":"100M","numThreads":3},
* {"name":"medium", "type": "internal", "maxSize":"500M","numThreads":3},
- * {"name": "large", "type": "external", "queue", "Queue1"}
+ * {"name":"large", "type":"external", "queue":"Queue1"}
* ]}
*
*
(accumulo) branch main updated: Renames split property to avoid tserver (#4450)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new c39599d75d Renames split property to avoid tserver (#4450)
c39599d75d is described below
commit c39599d75d9eed7a00d2bb506d2a3a35090be00a
Author: Keith Turner
AuthorDate: Thu Apr 11 19:17:43 2024 -0400
Renames split property to avoid tserver (#4450)
In 4.0.0 split code will no longer run in the tablet server. This
change deprecates a split related property that includes tablet server
in the property perfix and replaces it with a property that does
not include tablet sever.
---
.../main/java/org/apache/accumulo/core/conf/Property.java | 11 +++
.../main/java/org/apache/accumulo/server/util/FileUtil.java | 13 +
2 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index e2afb7defb..f86e4cad3b 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -395,6 +395,15 @@ public enum Property {
+ "indefinitely. Default is 0 to block indefinitely. Only valid when
tserver available "
+ "threshold is set greater than 0.",
"1.10.0"),
+ SPLIT_PREFIX("split.", null, PropertyType.PREFIX,
+ "System wide properties related to splitting tablets.", "3.1.0"),
+ SPLIT_MAXOPEN("split.files.max", "300", PropertyType.COUNT,
+ "To find a tablets split points, all RFiles are opened and their indexes"
+ + " are read. This setting determines how many RFiles can be opened
at once."
+ + " When there are more RFiles than this setting multiple passes
must be"
+ + " made, which is slower. However opening too many RFiles at once
can cause"
+ + " problems.",
+ "3.1.0"),
// properties that are specific to scan server behavior
@Experimental
SSERV_PREFIX("sserver.", null, PropertyType.PREFIX,
@@ -496,6 +505,8 @@ public enum Property {
TSERV_TOTAL_MUTATION_QUEUE_MAX("tserver.total.mutation.queue.max", "5%",
PropertyType.MEMORY,
"The amount of memory used to store write-ahead-log mutations before
flushing them.",
"1.7.0"),
+ @ReplacedBy(property = SPLIT_MAXOPEN)
+ @Deprecated(since = "3.1")
TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN("tserver.tablet.split.midpoint.files.max",
"300",
PropertyType.COUNT,
"To find a tablets split points, all RFiles are opened and their indexes"
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index a600dff94b..dd1dd91ed2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -33,6 +33,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.stream.Collectors;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -214,8 +215,7 @@ public class FileUtil {
Path tmpDir = null;
-int maxToOpen =
-
context.getConfiguration().getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN);
+int maxToOpen = getMaxFilesToOpen(context.getConfiguration());
ArrayList readers = new ArrayList<>(dataFiles.size());
try {
@@ -276,6 +276,12 @@ public class FileUtil {
}
}
+ @SuppressWarnings("deprecation")
+ private static int getMaxFilesToOpen(AccumuloConfiguration conf) {
+return conf.getCount(
+conf.resolve(Property.SPLIT_MAXOPEN,
Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN));
+ }
+
/**
*
* @param dataFiles - list of data files to find the mid point key
@@ -294,8 +300,7 @@ public class FileUtil {
Path tmpDir = null;
-int maxToOpen =
-
context.getConfiguration().getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN);
+int maxToOpen = getMaxFilesToOpen(context.getConfiguration());
ArrayList readers = new ArrayList<>(dataFiles.size());
try {
(accumulo) branch main updated: falls back to deprecated compaction prop (#4451)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 61e81c43d8 falls back to deprecated compaction prop (#4451)
61e81c43d8 is described below
commit 61e81c43d852be13f270d8e0638d6b4b9b4c5510
Author: Keith Turner
AuthorDate: Thu Apr 11 19:17:22 2024 -0400
falls back to deprecated compaction prop (#4451)
---
.../org/apache/accumulo/server/compaction/CompactionWatcher.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionWatcher.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionWatcher.java
index 4790440ff6..ff90bd5880 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionWatcher.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionWatcher.java
@@ -62,6 +62,12 @@ public class CompactionWatcher implements Runnable {
this.config = config;
}
+ @SuppressWarnings("deprecation")
+ private static long getCompactionWarnTime(AccumuloConfiguration config) {
+return config.getTimeInMillis(
+config.resolve(Property.COMPACTION_WARN_TIME,
Property.TSERV_COMPACTION_WARN_TIME));
+ }
+
@Override
public void run() {
List runningCompactions =
FileCompactor.getRunningCompactions();
@@ -98,7 +104,7 @@ public class CompactionWatcher implements Runnable {
// remove any compaction that completed or made progress
observedCompactions.keySet().retainAll(newKeys);
-long warnTime = config.getTimeInMillis(Property.COMPACTION_WARN_TIME);
+long warnTime = getCompactionWarnTime(config);
// check for stuck compactions
for (ObservedCompactionInfo oci : observedCompactions.values()) {
(accumulo) branch elasticity updated: fixes conditional mutation bug when loading tablet (#4447)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f2416680de fixes conditional mutation bug when loading tablet (#4447)
f2416680de is described below
commit f2416680de2f1923800a2c81c75ffefcf952d673
Author: Keith Turner
AuthorDate: Thu Apr 11 19:18:00 2024 -0400
fixes conditional mutation bug when loading tablet (#4447)
In #4436 a tablet update was moved to use conditional mutations. The
changed checked tablets current location in the conditional update.
However when this code was called in the Tablet constructor the tablet
server was the future location. Adjusted to code to check for the
future location when calling from the constructor.
---
.../java/org/apache/accumulo/tserver/tablet/ScanfileManager.java| 5 ++---
.../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java| 6 --
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
index 5ab4780a39..a0b00127af 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
@@ -144,7 +144,7 @@ class ScanfileManager {
}
}
- void removeFilesAfterScan(Collection scanFiles) {
+ void removeFilesAfterScan(Collection scanFiles, Location
location) {
if (scanFiles.isEmpty()) {
return;
}
@@ -163,8 +163,7 @@ class ScanfileManager {
if (!filesToDelete.isEmpty()) {
log.debug("Removing scan refs from metadata {} {}", tablet.getExtent(),
filesToDelete);
- var currLoc =
Location.current(tablet.getTabletServer().getTabletSession());
- removeScanFiles(tablet.getExtent(), filesToDelete, tablet.getContext(),
currLoc,
+ removeScanFiles(tablet.getExtent(), filesToDelete, tablet.getContext(),
location,
tablet.getTabletServer().getLock());
}
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 0565a00bbf..2e87b25305 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -326,7 +326,8 @@ public class Tablet extends TabletBase {
computeNumEntries();
-getScanfileManager().removeFilesAfterScan(metadata.getScans());
+getScanfileManager().removeFilesAfterScan(metadata.getScans(),
+Location.future(tabletServer.getTabletSession()));
}
public TabletMetadata getMetadata() {
@@ -1614,7 +1615,8 @@ public class Tablet extends TabletBase {
}
if (refreshPurpose == RefreshPurpose.REFRESH_RPC) {
- scanfileManager.removeFilesAfterScan(getMetadata().getScans());
+ scanfileManager.removeFilesAfterScan(getMetadata().getScans(),
+ Location.current(tabletServer.getTabletSession()));
}
}
(accumulo) branch elasticity updated: removes TODO that is done (#4448)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 4e8440a0cc removes TODO that is done (#4448)
4e8440a0cc is described below
commit 4e8440a0ccb449edc26dadfae6f2c01bf977627e
Author: Keith Turner
AuthorDate: Wed Apr 10 22:27:01 2024 -0400
removes TODO that is done (#4448)
This todo was already done in #4072
---
.../tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 414fedd105..0565a00bbf 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1647,7 +1647,6 @@ public class Tablet extends TabletBase {
getTabletMemory().getCommitSession().updateMaxCommittedTime(timestamp);
-// ELASTICITY_TODO this needs to be persisted in the metadata table or
walog
return OptionalLong.of(timestamp);
}
}
(accumulo) branch elasticity updated: updates bulk import and compaction logging (#4440)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 212e3cd27c updates bulk import and compaction logging (#4440)
212e3cd27c is described below
commit 212e3cd27cf9701e750ca8c3a2566a74d88867f3
Author: Keith Turner
AuthorDate: Wed Apr 10 10:56:01 2024 -0400
updates bulk import and compaction logging (#4440)
---
.../coordinator/commit/CommitCompaction.java | 20
.../manager/tableOps/bulkVer2/LoadFiles.java | 54 +-
.../manager/tableOps/compact/CompactionDriver.java | 2 -
3 files changed, 41 insertions(+), 35 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java
index cc0432d4a0..7add060466 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java
@@ -130,10 +130,11 @@ public class CommitCompaction extends ManagerRepo {
tabletMutator
.submit(tabletMetadata ->
!tabletMetadata.getExternalCompactions().containsKey(ecid));
-// TODO expensive logging
-LOG.debug("Compaction completed {} added {} removed {}",
tablet.getExtent(), newDatafile,
-ecm.getJobFiles().stream().map(AbstractTabletFile::getFileName)
-.collect(Collectors.toList()));
+if (LOG.isDebugEnabled()) {
+ LOG.debug("Compaction completed {} added {} removed {}",
tablet.getExtent(), newDatafile,
+ ecm.getJobFiles().stream().map(AbstractTabletFile::getFileName)
+ .collect(Collectors.toList()));
+}
var result = tabletsMutator.process().get(getExtent());
if (result.getStatus() == Ample.ConditionalResult.Status.ACCEPTED) {
@@ -159,7 +160,7 @@ public class CommitCompaction extends ManagerRepo {
private void updateTabletForCompaction(TCompactionStats stats,
ExternalCompactionId ecid,
TabletMetadata tablet, Optional newDatafile,
CompactionMetadata ecm,
Ample.ConditionalTabletMutator tabletMutator) {
-// ELASTICITY_TODO improve logging adapt to use existing tablet files
logging
+
if (ecm.getKind() == CompactionKind.USER) {
if (tablet.getSelectedFiles().getFiles().equals(ecm.getJobFiles())) {
// all files selected for the user compactions are finished, so the
tablet is finish and
@@ -171,8 +172,7 @@ public class CommitCompaction extends ManagerRepo {
"Tablet %s unexpected has selected files and compacted columns for
%s",
tablet.getExtent(), fateId);
-// TODO set to trace
-LOG.debug("All selected files compacted for {} setting compacted for
{}",
+LOG.trace("All selected files compacted for {} setting compacted for
{}",
tablet.getExtent(), tablet.getSelectedFiles().getFateId());
tabletMutator.deleteSelectedFiles();
@@ -187,14 +187,12 @@ public class CommitCompaction extends ManagerRepo {
newSelectedFileSet.removeAll(ecm.getJobFiles());
if (newDatafile.isPresent()) {
- // TODO set to trace
- LOG.debug(
+ LOG.trace(
"Not all selected files for {} are done, adding new selected
file {} from compaction",
tablet.getExtent(),
newDatafile.orElseThrow().getPath().getName());
newSelectedFileSet.add(newDatafile.orElseThrow().insert());
} else {
- // TODO set to trace
- LOG.debug(
+ LOG.trace(
"Not all selected files for {} are done, compaction produced no
output so not adding to selected set.",
tablet.getExtent());
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
index d336358700..1689ae4359 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import org.apache.accumulo.core.clientImpl.bulk.Bulk;
@@ -44,9 +45,11 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apa
(accumulo) branch elasticity updated: verifies tablets are seen by compaction driver (#4434)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 9503ebe22f verifies tablets are seen by compaction driver (#4434)
9503ebe22f is described below
commit 9503ebe22fef88ee3ecfbecbfecb9f5bfcc86e9b
Author: Keith Turner
AuthorDate: Mon Apr 8 12:37:39 2024 -0400
verifies tablets are seen by compaction driver (#4434)
---
.../accumulo/manager/tableOps/compact/CompactionDriver.java | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
index 0f224736a0..3790d981c3 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
@@ -61,10 +61,13 @@ import
org.apache.accumulo.manager.tableOps.delete.PreDeleteTable;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.compaction.CompactionConfigStorage;
import org.apache.accumulo.server.compaction.CompactionPluginUtils;
+import org.apache.hadoop.io.Text;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
class CompactionDriver extends ManagerRepo {
private static final Logger log =
LoggerFactory.getLogger(CompactionDriver.class);
@@ -300,6 +303,12 @@ class CompactionDriver extends ManagerRepo {
long t2 = System.currentTimeMillis();
+// The Fate operation gets a table lock that prevents the table from being
deleted while this is
+// running, so seeing zero tablets in the metadata table is unexpected.
+Preconditions.checkState(total > 0,
+"No tablets were seen for table %s in the compaction range %s %s",
tableId,
+new Text(startRow), new Text(endRow));
+
log.debug(
"{} tablet stats, total:{} complete:{} selected_now:{}
selected_prev:{} selected_by_other:{} "
+ "no_files:{} none_selected:{} user_compaction_requested:{}
user_compaction_waiting:{} "
@@ -314,8 +323,6 @@ class CompactionDriver extends ManagerRepo {
}
return total - complete;
-
-// ELASTICITIY_TODO need to handle seeing zero tablets
}
@Override
(accumulo) branch elasticity updated: conditionally removes tables scan files (#4436)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5a669c9d1f conditionally removes tables scan files (#4436)
5a669c9d1f is described below
commit 5a669c9d1fb36e3989da0b2c7b3889fd9f496443
Author: Keith Turner
AuthorDate: Mon Apr 8 12:36:20 2024 -0400
conditionally removes tables scan files (#4436)
---
.../accumulo/server/util/MetadataTableUtil.java| 9 --
.../accumulo/tserver/tablet/ScanfileManager.java | 35 ++
2 files changed, 30 insertions(+), 14 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index e5db26344f..53dc257ef9 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -37,7 +37,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
@@ -152,14 +151,6 @@ public class MetadataTableUtil {
return newFiles;
}
- public static void removeScanFiles(KeyExtent extent, Set
scanFiles,
- ServerContext context, ServiceLock zooLock) {
-TabletMutator tablet = context.getAmple().mutateTablet(extent);
-scanFiles.forEach(tablet::deleteScan);
-tablet.putZooLock(context.getZooKeeperRoot(), zooLock);
-tablet.mutate();
- }
-
public static void deleteTable(TableId tableId, boolean insertDeletes,
ServerContext context,
ServiceLock lock) throws AccumuloException {
try (
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
index e53bbf2553..5ab4780a39 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanfileManager.java
@@ -20,21 +20,28 @@ package org.apache.accumulo.tserver.tablet;
import java.io.IOException;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.util.MapCounter;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
class ScanfileManager {
private final Logger log = LoggerFactory.getLogger(ScanfileManager.class);
private final Tablet tablet;
@@ -54,6 +61,23 @@ class ScanfileManager {
}
}
+ static void removeScanFiles(KeyExtent extent, Set
scanFiles,
+ ServerContext context, Location currLocation, ServiceLock zooLock) {
+try (var mutator = context.getAmple().conditionallyMutateTablets()) {
+ var tabletMutator =
mutator.mutateTablet(extent).requireLocation(currLocation);
+
+ scanFiles.forEach(tabletMutator::deleteScan);
+ tabletMutator.putZooLock(context.getZooKeeperRoot(), zooLock);
+
+ tabletMutator
+ .submit(tabletMetadata -> Collections.disjoint(scanFiles,
tabletMetadata.getScans()));
+
+ var result = mutator.process().get(extent);
+ Preconditions.checkState(result.getStatus() ==
Ample.ConditionalResult.Status.ACCEPTED,
+ "Failed to remove scan file entries for %s", extent);
+}
+ }
+
Pair> reserveFilesForScan() {
synchronized (tablet) {
@@ -112,8 +136,9 @@ class ScanfileManager {
// file is in the set filesToDelete that means it was removed from
filesToDeleteAfterScan
// and would never be added back.
log.debug("Removing scan refs from metadata {} {}",
tablet.getExtent(), filesToDelete);
-// ELASTICTIY_TODO use conditional mutation
-MetadataTableUtil.removeScanFiles(tablet.getExtent(), filesToDelete,
tablet.getContext(),
+
+var currLoc =
Location.current(tablet.getTabletServer().getTabletSession());
+removeScanFiles(tablet.getExtent(), filesToDelete,
ta
(accumulo) branch elasticity updated (0386506c1d -> cb097aee5f)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 0386506c1d moves ample filters out of public API (#4431) add b912506d43 avoids acquiring recovery lock when tablet has no wals (#4429) new cb097aee5f Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/AssignmentHandler.java | 6 +- .../java/org/apache/accumulo/tserver/TabletServer.java | 16 2 files changed, 9 insertions(+), 13 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit cb097aee5f36fb91f04c09db084595ef624cd2c8
Merge: 0386506c1d b912506d43
Author: Keith Turner
AuthorDate: Fri Apr 5 15:19:05 2024 -0400
Merge branch 'main' into elasticity
.../org/apache/accumulo/tserver/AssignmentHandler.java | 6 +-
.../java/org/apache/accumulo/tserver/TabletServer.java | 16
2 files changed, 9 insertions(+), 13 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
index cef355cbf1,552d9f40a9..ec56611a03
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
@@@ -131,13 -156,12 +131,11 @@@ class AssignmentHandler implements Runn
Tablet tablet = null;
boolean successful = false;
- try {
- server.acquireRecoveryMemory(extent);
-
+ try (var recoveryMemory = server.acquireRecoveryMemory(tabletMetadata)) {
TabletResourceManager trm =
server.resourceManager.createTabletResourceManager(extent,
server.getTableConfiguration(extent));
- TabletData data = new TabletData(tabletMetadata);
- tablet = new Tablet(server, extent, trm, data);
+ tablet = new Tablet(server, extent, trm, tabletMetadata);
// If a minor compaction starts after a tablet opens, this indicates a
log recovery
// occurred. This recovered data must be minor compacted.
// There are three reasons to wait for this minor compaction to finish
before placing the
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 63fcbd4c5b,678b1294c5..026c904ca2
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -87,7 -91,8 +87,8 @@@ import org.apache.accumulo.core.manager
import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
+ import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
(accumulo) branch main updated: avoids acquiring recovery lock when tablet has no wals (#4429)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new b912506d43 avoids acquiring recovery lock when tablet has no wals
(#4429)
b912506d43 is described below
commit b912506d43fbac9477aa3efb9ea81d39ce4d1a5c
Author: Keith Turner
AuthorDate: Fri Apr 5 13:01:07 2024 -0400
avoids acquiring recovery lock when tablet has no wals (#4429)
Tablet servers have a lock for recovery that has the purpose of
preventing concurrent recovery of tablets using too much memory.
This lock is acquired for tablets that have no walogs and will therefore
do no recovery. This commit changes the code to only obtain the lock
when there are walogs.
Noticed this while reviewing #4404 and while researching found
[ACCUMULO-1543](https://issues.apache.org/jira/browse/ACCUMULO-1543)
which is an existing issue describing this change.
---
.../org/apache/accumulo/tserver/AssignmentHandler.java | 6 +-
.../java/org/apache/accumulo/tserver/TabletServer.java | 16
2 files changed, 9 insertions(+), 13 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
index 8b4f117a21..552d9f40a9 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/AssignmentHandler.java
@@ -156,9 +156,7 @@ class AssignmentHandler implements Runnable {
Tablet tablet = null;
boolean successful = false;
-try {
- server.acquireRecoveryMemory(extent);
-
+try (var recoveryMemory = server.acquireRecoveryMemory(tabletMetadata)) {
TabletResourceManager trm =
server.resourceManager.createTabletResourceManager(extent,
server.getTableConfiguration(extent));
TabletData data = new TabletData(tabletMetadata);
@@ -205,8 +203,6 @@ class AssignmentHandler implements Runnable {
TableId tableId = extent.tableId();
ProblemReports.getInstance(server.getContext()).report(new
ProblemReport(tableId, TABLET_LOAD,
extent.getUUID().toString(), server.getClientAddressString(), e));
-} finally {
- server.releaseRecoveryMemory(extent);
}
if (successful) {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index b9c9a39359..678b1294c5 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -91,6 +91,7 @@ import org.apache.accumulo.core.manager.thrift.TableInfo;
import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.rpc.ThriftUtil;
@@ -516,15 +517,14 @@ public class TabletServer extends AbstractServer
implements TabletHostingServer
managerMessages.addLast(m);
}
- void acquireRecoveryMemory(KeyExtent extent) {
-if (!extent.isMeta()) {
- recoveryLock.lock();
-}
- }
+ private static final AutoCloseable NOOP_CLOSEABLE = () -> {};
- void releaseRecoveryMemory(KeyExtent extent) {
-if (!extent.isMeta()) {
- recoveryLock.unlock();
+ AutoCloseable acquireRecoveryMemory(TabletMetadata tabletMetadata) {
+if (tabletMetadata.getExtent().isMeta() ||
tabletMetadata.getLogs().isEmpty()) {
+ return NOOP_CLOSEABLE;
+} else {
+ recoveryLock.lock();
+ return recoveryLock::unlock;
}
}
(accumulo) branch elasticity updated: moves ample filters out of public API (#4431)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 0386506c1d moves ample filters out of public API (#4431)
0386506c1d is described below
commit 0386506c1d91a5edfcc524df17ee472992adc46d
Author: Keith Turner
AuthorDate: Fri Apr 5 12:05:38 2024 -0400
moves ample filters out of public API (#4431)
Ample filters were in a package that is in the public API. Since Ample
itself is not in the public API, these filters should also not be in the
public API. This commit moves the filters to another packages that is
not in the public API
---
.../org/apache/accumulo/core/metadata/schema/TabletsMetadata.java | 2 +-
.../{iterators/user => metadata/schema/filters}/GcWalsFilter.java | 5 +
.../user => metadata/schema/filters}/HasCurrentFilter.java | 2 +-
.../schema/filters}/HasExternalCompactionsFilter.java | 2 +-
.../user => metadata/schema/filters}/TabletMetadataFilter.java | 3 ++-
.../java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java | 2 +-
.../manager/compaction/coordinator/CompactionCoordinator.java | 2 +-
.../manager/compaction/coordinator/DeadCompactionDetector.java | 2 +-
.../org/apache/accumulo/monitor/rest/tables/TablesResource.java | 2 +-
.../apache/accumulo/test/functional/AmpleConditionalWriterIT.java | 6 +++---
.../apache/accumulo/test/functional/TestTabletMetadataFilter.java | 2 +-
11 files changed, 14 insertions(+), 16 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 14b8181e78..48a1160513 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -63,7 +63,6 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.fate.zookeeper.ZooCache;
import org.apache.accumulo.core.fate.zookeeper.ZooReader;
-import org.apache.accumulo.core.iterators.user.TabletMetadataFilter;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.RootTable;
@@ -85,6 +84,7 @@ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Sp
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.filters.TabletMetadataFilter;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.hadoop.io.Text;
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/user/GcWalsFilter.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/filters/GcWalsFilter.java
similarity index 94%
rename from
core/src/main/java/org/apache/accumulo/core/iterators/user/GcWalsFilter.java
rename to
core/src/main/java/org/apache/accumulo/core/metadata/schema/filters/GcWalsFilter.java
index a30302ce59..4e1aca1a06 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/user/GcWalsFilter.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/filters/GcWalsFilter.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.core.iterators.user;
+package org.apache.accumulo.core.metadata.schema.filters;
import java.io.IOException;
import java.util.Arrays;
@@ -41,9 +41,6 @@ import com.google.common.collect.Sets;
* A filter used by the Accumulo GC to find tablets that either have walogs or
are assigned to a
* dead tablet server.
*/
-
-// ELASTICITY_TODO Move TabletMetadataFilter and its subclasses out of public
API. It use internal
-// types that are not user facing.
public class GcWalsFilter extends TabletMetadataFilter {
private Map options = null;
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/user/HasCurrentFilter.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/filters/HasCurrentFilter.java
similarity index 96%
rename from
core/src/main/java/org/apache/accumulo/core/iterators/user/HasCurrentFilter.java
rename to
core/src/main/java/org/apache/accumulo/core/metadata/schema/filters/HasCurrentFilter.java
index ca58922306..0d00b59837 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/user/HasCurrentFilter.java
+++
b/core
(accumulo) branch elasticity updated: avoid unnecessary metadata reads in client tablet cache (#4432)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f586e7f9c9 avoid unnecessary metadata reads in client tablet cache
(#4432)
f586e7f9c9 is described below
commit f586e7f9c9a82c0f01cba1ac279a7d583c7be3c4
Author: Keith Turner
AuthorDate: Fri Apr 5 12:05:21 2024 -0400
avoid unnecessary metadata reads in client tablet cache (#4432)
For ondemand tablets the client tablet cache caches tablets w/o a
location. There was a bug fixed in #4280 where the cache would do a
metadata table lookup for each mutation when tablets had no location.
The fix in #4280 only partially fixed the problem, after that change
more metadata lookups than needed were still being done.
Also there was a bug with the batchscanner that #4280 did not address.
Before this change when tablets had no location, the batch scanner would
do a metadata lookup for each range passed to the batch scanner (well
the client tablet cache would these metadata lookups on behalf of the
batch scanner). For example before this change if the batch scanner was
given 10K ranges that all fell in a single tablet w/o a location, it
would do 10K metadata lookups. After this change for that situation it
will do a single metadata lookup.
This change minimizes the metadata lookups done by the batch writer and
batch scanner. The fix is to make sure that cached entries populated by
looking up one range or mutation are used by subsequent range or
mutations lookups, even if there is no location present. This is done by
always reusing cache entries that were created after work started on a
batch of mutations or ranges. Cache entries w/o a location that existed
before work started on a batch are ignored. By reusing cache entries
created after starting work on a batch we minimize metadata lookups.
A test was also added to ensure the client tablet cache does not do
excessive metadata table lookups. If this test had existed, it would
have caught the problem.
---
.../core/clientImpl/ClientTabletCache.java | 8 +-
.../core/clientImpl/ClientTabletCacheImpl.java | 66 ++
.../core/clientImpl/ClientTabletCacheImplTest.java | 137 +
3 files changed, 181 insertions(+), 30 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
index 2564fb0063..a31ca2418b 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
@@ -20,7 +20,6 @@ package org.apache.accumulo.core.clientImpl;
import static com.google.common.base.Preconditions.checkArgument;
-import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -46,6 +45,7 @@ import org.apache.accumulo.core.singletons.SingletonManager;
import org.apache.accumulo.core.singletons.SingletonService;
import org.apache.accumulo.core.util.Interner;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.time.NanoTime;
import org.apache.hadoop.io.Text;
import com.google.common.base.Preconditions;
@@ -311,7 +311,7 @@ public abstract class ClientTabletCache {
private final TabletAvailability availability;
private final boolean hostingRequested;
-private final Long creationTime = System.nanoTime();
+private final NanoTime creationTime = NanoTime.now();
public CachedTablet(KeyExtent tablet_extent, String tablet_location,
String session,
TabletAvailability availability, boolean hostingRequested) {
@@ -392,8 +392,8 @@ public abstract class ClientTabletCache {
return this.availability;
}
-public Duration getAge() {
- return Duration.ofNanos(System.nanoTime() - creationTime);
+public NanoTime getCreationTime() {
+ return creationTime;
}
public boolean wasHostingRequested() {
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
index 10fb3aa21e..eb5225 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
@@ -60,6 +60,7 @@ import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.util.time.NanoTime;
import org.apache.hadoop.io.Text;
import
(accumulo) branch elasticity updated (cebef4bdbc -> ebfa35b49e)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from cebef4bdbc cleaned up a few TODOs in the split code (#4401) add 7947c2c2df adds PluginEnv support to client side iterators (#4283) add eaa1f678c9 Merge branch '2.1' add ebfa35b49e Merge branch 'main' into elasticity No new revisions were added by this update. Summary of changes: .../core/client/ClientSideIteratorScanner.java | 39 +++- .../accumulo/core/clientImpl/OfflineIterator.java | 27 +- .../accumulo/core/clientImpl/ScannerImpl.java | 10 ++ .../apache/accumulo/test/ClientSideIteratorIT.java | 108 + 4 files changed, 178 insertions(+), 6 deletions(-)
(accumulo) branch main updated (0ad96b1dc0 -> eaa1f678c9)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 0ad96b1dc0 Merge remote-tracking branch 'upstream/2.1' add 7947c2c2df adds PluginEnv support to client side iterators (#4283) new eaa1f678c9 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/ClientSideIteratorScanner.java | 39 +++- .../accumulo/core/clientImpl/OfflineIterator.java | 27 +- .../accumulo/core/clientImpl/ScannerImpl.java | 10 ++ .../apache/accumulo/test/ClientSideIteratorIT.java | 108 + 4 files changed, 178 insertions(+), 6 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit eaa1f678c96b135534fba89fe585107c4f242e54
Merge: 0ad96b1dc0 7947c2c2df
Author: Keith Turner
AuthorDate: Wed Mar 20 18:39:26 2024 -0400
Merge branch '2.1'
.../core/client/ClientSideIteratorScanner.java | 39 +++-
.../accumulo/core/clientImpl/OfflineIterator.java | 27 +-
.../accumulo/core/clientImpl/ScannerImpl.java | 10 ++
.../apache/accumulo/test/ClientSideIteratorIT.java | 108 +
4 files changed, 178 insertions(+), 6 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index a50561263f,362c4e85d6..7e300e917d
---
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@@ -122,6 -133,22 +133,16 @@@ public class ClientSideIteratorScanner
public SamplerConfiguration getSamplerConfiguration() {
return samplerConfig;
}
+
-@Deprecated(since = "2.1.0")
-@Override
-public ServiceEnvironment getServiceEnv() {
- return new ClientServiceEnvironmentImpl(context.get());
-}
-
+ @Override
+ public PluginEnvironment getPluginEnv() {
+ return new ClientServiceEnvironmentImpl(context.get());
+ }
+
+ @Override
+ public TableId getTableId() {
+ return tableId.get();
+ }
}
/**
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
index 0e7a3fd4d2,a03cc811ab..9200591ff2
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
@@@ -135,7 -153,24 +140,18 @@@ class OfflineIterator implements Iterat
if (sampleConf == null) {
throw new SampleNotPresentException();
}
- return new OfflineIteratorEnvironment(authorizations, conf, true,
sampleConf);
+ return new OfflineIteratorEnvironment(context, tableId, authorizations,
conf, true,
+ sampleConf);
+ }
+
-@Deprecated(since = "2.1.0")
-@Override
-public ServiceEnvironment getServiceEnv() {
- return new ClientServiceEnvironmentImpl(context);
-}
-
+ @Override
+ public PluginEnvironment getPluginEnv() {
+ return new ClientServiceEnvironmentImpl(context);
+ }
+
+ @Override
+ public TableId getTableId() {
+ return tableId;
}
}
(accumulo) branch elasticity updated: cleaned up a few TODOs in the split code (#4401)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new cebef4bdbc cleaned up a few TODOs in the split code (#4401)
cebef4bdbc is described below
commit cebef4bdbce59c9f33da8409d64b590b59166195
Author: Keith Turner
AuthorDate: Wed Mar 20 18:26:20 2024 -0400
cleaned up a few TODOs in the split code (#4401)
Removed TODOs in the code related to #3415, #3709, #3412. These todos
were done or had issues opened.
Added some best effot checks to avoid doing splits if a table is
offline. The checks have race conditions as the table lock is not
acquired by the split fate op. However as outlined in a comment
on #3412 the offline code that waits could be made more
comprehensive to account for this.
One TODO was handled by using
`Ample.RejectionHandler.acceptAbsentTablet()` which likely did not
exists when the TODO was written.
---
.../accumulo/manager/FateServiceHandler.java | 6 --
.../manager/tableOps/split/FindSplits.java | 7 +++
.../accumulo/manager/tableOps/split/PreSplit.java | 20 ---
.../manager/tableOps/split/UpdateTablets.java | 23 +++---
4 files changed, 27 insertions(+), 29 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
index 5c2ae952d0..3e45faca23 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
@@ -719,10 +719,6 @@ class FateServiceHandler implements FateService.Iface {
case TABLE_SPLIT: {
TableOperation tableOp = TableOperation.SPLIT;
-// ELASTICITY_TODO this does not check if table is offline for now,
that is usually done in
-// FATE operation with a table lock. Deferring that check for now as
its possible tablet
-// locks may not be needed.
-
int SPLIT_OFFSET = 3; // offset where split data begins in arguments
list
if (arguments.size() < (SPLIT_OFFSET + 1)) {
throw new ThriftTableOperationException(null, null, tableOp,
@@ -752,8 +748,6 @@ class FateServiceHandler implements FateService.Iface {
endRow = endRow.getLength() == 0 ? null : endRow;
prevEndRow = prevEndRow.getLength() == 0 ? null : prevEndRow;
-// ELASTICITY_TODO create table stores splits in a file, maybe this
operation should do the
-// same
SortedSet splits = arguments.subList(SPLIT_OFFSET,
arguments.size()).stream()
.map(ByteBufferUtil::toText).collect(Collectors.toCollection(TreeSet::new));
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java
index 7e3acd8d7f..f72d26cb05 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java
@@ -28,6 +28,7 @@ import java.util.function.Consumer;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult;
import org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult.Status;
import org.apache.accumulo.core.metadata.schema.UnSplittableMetadata;
@@ -88,6 +89,12 @@ public class FindSplits extends ManagerRepo {
return null;
}
+if (manager.getContext().getTableState(extent.tableId()) !=
TableState.ONLINE) {
+ // The table is offline, do not bother finding splits
+ log.debug("Not splitting {} because the table is not online",
tabletMetadata.getExtent());
+ return null;
+}
+
SortedSet splits = SplitUtils.findSplits(manager.getContext(),
tabletMetadata);
if (extent.endRow() != null) {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
index d956821d18..2bf5007d8c 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
@@ -30,9 +30,13 @@ import java.util.Objects;
import java.util.Optional;
import java.util.SortedSet;
+import
org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationExcep
(accumulo) branch 2.1 updated: adds PluginEnv support to client side iterators (#4283)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 7947c2c2df adds PluginEnv support to client side iterators (#4283)
7947c2c2df is described below
commit 7947c2c2dfb80fb280420b729b4197547be22124
Author: Keith Turner
AuthorDate: Wed Mar 20 18:25:43 2024 -0400
adds PluginEnv support to client side iterators (#4283)
Accumulo code that ran user iterators client side would throw an unsupported
operations exception when attempting to access the PluginEnv. This commit
adds support for PluginEnv in situations where iterators are run client
side.
---
.../core/client/ClientSideIteratorScanner.java | 46 -
.../accumulo/core/clientImpl/OfflineIterator.java | 34 ++-
.../accumulo/core/clientImpl/ScannerImpl.java | 10 ++
.../apache/accumulo/test/ClientSideIteratorIT.java | 108 +
4 files changed, 192 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index 542273d470..362c4e85d6 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@ -29,15 +29,20 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.function.Supplier;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl;
+import org.apache.accumulo.core.clientImpl.ScannerImpl;
import org.apache.accumulo.core.clientImpl.ScannerOptions;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Column;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
import org.apache.accumulo.core.iterators.IteratorAdapter;
@@ -47,6 +52,7 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder;
import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.hadoop.io.Text;
/**
@@ -70,6 +76,7 @@ import org.apache.hadoop.io.Text;
* server side) and to the client side scanner (which will execute client
side).
*/
public class ClientSideIteratorScanner extends ScannerOptions implements
Scanner {
+
private int size;
private Range range;
@@ -77,6 +84,9 @@ public class ClientSideIteratorScanner extends ScannerOptions
implements Scanner
private long readaheadThreshold =
Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD;
private SamplerConfiguration iteratorSamplerConfig;
+ private final Supplier context;
+ private final Supplier tableId;
+
private class ClientSideIteratorEnvironment implements IteratorEnvironment {
private SamplerConfiguration samplerConfig;
@@ -94,7 +104,9 @@ public class ClientSideIteratorScanner extends
ScannerOptions implements Scanner
@Override
public boolean isFullMajorCompaction() {
- return false;
+ // The javadocs state this method will throw an ISE when scope is not
majc
+ throw new IllegalStateException(
+ "Asked about major compaction type when scope is " +
getIteratorScope());
}
@Override
@@ -121,6 +133,22 @@ public class ClientSideIteratorScanner extends
ScannerOptions implements Scanner
public SamplerConfiguration getSamplerConfiguration() {
return samplerConfig;
}
+
+@Deprecated(since = "2.1.0")
+@Override
+public ServiceEnvironment getServiceEnv() {
+ return new ClientServiceEnvironmentImpl(context.get());
+}
+
+@Override
+public PluginEnvironment getPluginEnv() {
+ return new ClientServiceEnvironmentImpl(context.get());
+}
+
+@Override
+public TableId getTableId() {
+ return tableId.get();
+}
}
/**
@@ -220,6 +248,22 @@ public class ClientSideIteratorScanner extends
ScannerOptions implements Scanner
if (samplerConfig != null) {
setSamplerConfiguration(samplerConfig);
}
+
+if (scanner instanceof ScannerImpl) {
+ var scannerImpl = (ScannerImpl) scanner;
+ this.context = () -> scannerImpl.getClientContext();
+ this.tableId = () -&
(accumulo) branch elasticity updated: removed queued jobs for a tablet if no new jobs are seen (#4394)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 36df8cee36 removed queued jobs for a tablet if no new jobs are seen
(#4394)
36df8cee36 is described below
commit 36df8cee360081ca445f615cbc354e9e2ff2e6fc
Author: Keith Turner
AuthorDate: Mon Mar 18 13:51:20 2024 -0400
removed queued jobs for a tablet if no new jobs are seen (#4394)
fixes #3528
---
.../accumulo/manager/TabletGroupWatcher.java | 2 +
.../queue/CompactionJobPriorityQueue.java | 56 -
.../compaction/queue/CompactionJobQueues.java | 35 +++-
.../compaction/CompactionCoordinatorTest.java | 1 +
.../queue/CompactionJobPriorityQueueTest.java | 14 +-
.../compaction/queue/CompactionJobQueuesTest.java | 229 -
6 files changed, 314 insertions(+), 23 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 8779568916..3e8f1fdc5b 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -678,7 +678,9 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
eventHandler.clearNeedsFullScan();
iter = store.iterator(tableMgmtParams);
+
manager.getCompactionCoordinator().getJobQueues().beginFullScan(store.getLevel());
var tabletMgmtStats = manageTablets(iter, tableMgmtParams,
currentTServers, true);
+
manager.getCompactionCoordinator().getJobQueues().endFullScan(store.getLevel());
// If currently looking for volume replacements, determine if the next
round needs to look.
if (lookForTabletsNeedingVolReplacement) {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
index e4aa059e95..4dfd6868ad 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
@@ -23,18 +23,23 @@ import static
com.google.common.base.Preconditions.checkState;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactorGroupId;
import org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
@@ -49,6 +54,8 @@ import com.google.common.base.Preconditions;
*/
public class CompactionJobPriorityQueue {
+ private static final Logger log =
LoggerFactory.getLogger(CompactionJobPriorityQueue.class);
+
private final CompactorGroupId groupId;
private class CjpqKey implements Comparable {
@@ -99,9 +106,19 @@ public class CompactionJobPriorityQueue {
private final AtomicLong rejectedJobs;
private final AtomicLong dequeuedJobs;
+ private static class TabletJobs {
+final long generation;
+final HashSet jobs;
+
+private TabletJobs(long generation, HashSet jobs) {
+ this.generation = generation;
+ this.jobs = jobs;
+}
+ }
+
// This map tracks what jobs a tablet currently has in the queue. Its used
to efficiently remove
// jobs in the queue when new jobs are queued for a tablet.
- private final Map> tabletJobs;
+ private final Map tabletJobs;
private final AtomicLong nextSeq = new AtomicLong(0);
@@ -116,10 +133,32 @@ public class CompactionJobPriorityQueue {
this.dequeuedJobs = new AtomicLong(0);
}
+ public synchronized void removeOlderGenerations(Ample.DataLevel level, long
currGeneration) {
+if (closed.get()) {
+ return;
+}
+
+List removals = new ArrayList<>();
+
+tabletJobs.forEach((extent, jobs) -> {
+ if (Ample.DataLevel.of(extent.tableId()) == level && jobs.generation <
currGeneration) {
+removals.add(extent);
+ }
+});
+
+if (!removals.isEmpty()) {
+ log.trace("Removed {} queued
(accumulo) branch elasticity updated: forgo fabricating fake fateid for system compation (#4392)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f05e44335b forgo fabricating fake fateid for system compation (#4392)
f05e44335b is described below
commit f05e44335be91fc60097d336b98b3eed7f979a5c
Author: Keith Turner
AuthorDate: Sat Mar 16 14:52:44 2024 -0400
forgo fabricating fake fateid for system compation (#4392)
---
.../manager/compaction/coordinator/CompactionCoordinator.java | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 174d741abb..178b4f1e95 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -42,7 +42,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -584,8 +583,11 @@ public class CompactionCoordinator
dfv.getTime());
}).collect(toList());
-FateInstanceType type =
FateInstanceType.fromTableId(metaJob.getTabletMetadata().getTableId());
-FateId fateId = FateId.from(type, UUID.randomUUID());
+// The fateId here corresponds to the Fate transaction that is driving a
user initiated
+// compaction. A system initiated compaction has no Fate transaction
driving it so its ok to set
+// it to null. If anything tries to use the id for a system compaction and
triggers a NPE it's
+// probably a bug that needs to be fixed.
+FateId fateId = null;
if (metaJob.getJob().getKind() == CompactionKind.USER) {
fateId = metaJob.getTabletMetadata().getSelectedFiles().getFateId();
}
@@ -593,7 +595,8 @@ public class CompactionCoordinator
return new TExternalCompactionJob(externalCompactionId,
metaJob.getTabletMetadata().getExtent().toThrift(), files,
iteratorSettings,
ecm.getCompactTmpName().getNormalizedPathStr(),
ecm.getPropagateDeletes(),
-TCompactionKind.valueOf(ecm.getKind().name()), fateId.toThrift(),
overrides);
+TCompactionKind.valueOf(ecm.getKind().name()), fateId == null ? null :
fateId.toThrift(),
+overrides);
}
@Override
(accumulo) branch elasticity updated: Adds test for illegal fate ids (#4391)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 7d7c28c1a1 Adds test for illegal fate ids (#4391)
7d7c28c1a1 is described below
commit 7d7c28c1a133aebc8e914e8b27aabeb6632eeb59
Author: Keith Turner
AuthorDate: Sat Mar 16 14:52:19 2024 -0400
Adds test for illegal fate ids (#4391)
---
.../org/apache/accumulo/core/fate/FateIdTest.java | 66 ++
1 file changed, 66 insertions(+)
diff --git a/core/src/test/java/org/apache/accumulo/core/fate/FateIdTest.java
b/core/src/test/java/org/apache/accumulo/core/fate/FateIdTest.java
new file mode 100644
index 00..3de10825e2
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/fate/FateIdTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.fate;
+
+import static org.apache.accumulo.core.fate.FateInstanceType.META;
+import static org.apache.accumulo.core.fate.FateInstanceType.USER;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+public class FateIdTest {
+
+ @Test
+ public void testIllegal() {
+var uuid = UUID.randomUUID();
+
+var legalId = FateId.from(USER, uuid);
+
+// test the test assumptions about what is legal
+assertNotNull(FateId.from("FATE:" + META + ":" + uuid));
+assertEquals(legalId, FateId.from(legalId.canonical()));
+
+// The fate id has three fields, try making each field invalid and having
the wrong number of
+// fields
+for (var illegalId : new String[] {"1234567890", "FATE:" + uuid,
"FATE:GOLANG:" + uuid,
+META + ":" + uuid, USER + ":" + uuid, "RUST:" + META + ":" + uuid,
+"RUST:" + USER + ":" + uuid, legalId + ":JAVA", "JAVA:" + legalId,
"FATE:" + legalId}) {
+ assertThrows(IllegalArgumentException.class, () ->
FateId.from(illegalId));
+ assertFalse(FateId.isFateId(illegalId));
+}
+
+// Try different illegal uuids in the fate id
+for (var illegalUuid : List.of("3366-4485-92ab-6961bbd6d3f4",
"075cd820-4485-92ab-6961bbd6d3f4",
+"075cd820-3366-4485-92ab", "075C++20-3366-4485-92ab-6961bbd6d3f4",
+"075cd820--4485-92ab-6961bbd6d3f4",
"42b64e8-4307-4f7d-8466-a11e81eb56c7",
+"842b64e8-307-4f7d-8466-a11e81eb56c7",
"842b64e8-4307-f7d-8466-a11e81eb56c7",
+"842b64e8-4307-4f7d-466-a11e81eb56c7",
"842b64e8-4307-4f7d-8466-11e81eb56c7",
+"842b64e843074f7d8466a11e81eb56c7")) {
+ var illegalId = "FATE:" + USER + ":" + illegalUuid;
+ assertThrows(IllegalArgumentException.class, () ->
FateId.from(illegalId));
+ assertFalse(FateId.isFateId(illegalId));
+}
+ }
+}
(accumulo) branch elasticity updated: fixes compile error after clean merge of #4388 (#4390)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c5f4b9b678 fixes compile error after clean merge of #4388 (#4390)
c5f4b9b678 is described below
commit c5f4b9b678abdc993c0f32274b210212b2458710
Author: Keith Turner
AuthorDate: Sat Mar 16 13:02:07 2024 -0400
fixes compile error after clean merge of #4388 (#4390)
---
.../test/java/org/apache/accumulo/core/util/FastFormatTest.java| 7 +--
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
b/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
index 8e68d87a34..545f3feaf2 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
@@ -24,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.Arrays;
-import org.apache.accumulo.core.fate.FateId;
import org.junit.jupiter.api.Test;
public class FastFormatTest {
@@ -121,14 +120,10 @@ public class FastFormatTest {
@Test
public void testHexString() {
-String prefix = "FATE:USER:";
-assertEquals(FateId.formatTid(987654321L),
FastFormat.toHexString(987654321L));
-long txid = FateId.from(prefix + "2e429160071c63d8").getTid();
-assertEquals(prefix + "2e429160071c63d8", FastFormat.toHexString(prefix,
txid, ""));
assertEquals(String.format("%016x", 64L), FastFormat.toHexString(64L));
assertEquals(String.format("%016x", 0X2e429160071c63d8L),
FastFormat.toHexString(0X2e429160071c63d8L));
-
+assertEquals(String.format("%016x", 987654321L),
FastFormat.toHexString(987654321L));
assertEquals("-0040-", FastFormat.toHexString("-", 64L, "-"));
assertEquals("-075bcd15", FastFormat.toHexString("-", 123456789L,
""));
assertEquals("000a", FastFormat.toHexString(0XaL));
(accumulo) branch elasticity updated: FateId to use UUID instead of long (#4388)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new eb71e3e4a6 FateId to use UUID instead of long (#4388)
eb71e3e4a6 is described below
commit eb71e3e4a6af3c37ee27c9f5cb3b26277d74b05b
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Sat Mar 16 12:25:26 2024 -0400
FateId to use UUID instead of long (#4388)
- FateId now uses a 128bit UUID instead of a (64bit) long
- FateIds created by ZooStore and AccumuloStore now use a v4 UUID
(random UUID)
- FateIds created by FateIdGenerator now use a v3 UUID (name based)
so the same FateKey gives the same UUID
- Necessary updates to classes and tests which used the long id
---
.../accumulo/core/fate/AbstractFateStore.java | 12 +-
.../java/org/apache/accumulo/core/fate/FateId.java | 56 ---
.../org/apache/accumulo/core/fate/ZooStore.java| 8 +-
.../accumulo/core/fate/accumulo/AccumuloStore.java | 8 +-
.../core/fate/zookeeper/ZooReservation.java| 2 +-
.../accumulo/core/manager/thrift/TFateId.java | 125
core/src/main/thrift/manager.thrift| 2 +-
.../org/apache/accumulo/core/fate/TestStore.java | 4 +-
.../core/metadata/schema/SelectedFilesTest.java| 19 ++-
.../core/metadata/schema/TabletMetadataTest.java | 61
.../server/compaction/CompactionConfigStorage.java | 4 +-
.../constraints/MetadataConstraintsTest.java | 52 +++
.../server/manager/state/TabletManagementTest.java | 8 +-
.../server/util/fateCommand/SummaryReportTest.java | 3 +-
.../server/util/fateCommand/TxnDetailsTest.java| 14 +-
.../accumulo/manager/FateServiceHandler.java | 8 +-
.../coordinator/CompactionCoordinator.java | 3 +-
.../compaction/CompactionCoordinatorTest.java | 6 +-
.../manager/tableOps/ShutdownTServerTest.java | 3 +-
.../manager/tableOps/merge/MergeTabletsTest.java | 4 +-
.../manager/tableOps/split/UpdateTabletsTest.java | 16 +-
.../org/apache/accumulo/test/ScanServerIT.java | 3 +-
.../test/fate/accumulo/AccumuloFateIT.java | 2 +-
.../test/fate/accumulo/AccumuloStoreIT.java| 19 ++-
.../test/fate/accumulo/FateMutatorImplIT.java | 10 +-
.../accumulo/test/fate/accumulo/FateStoreIT.java | 12 +-
.../test/fate/zookeeper/ZooStoreFateIT.java| 2 +-
.../test/fate/zookeeper/ZookeeperFateIT.java | 2 +-
.../test/functional/AmpleConditionalWriterIT.java | 161 ++---
.../test/functional/ManagerAssignmentIT.java | 3 +-
.../apache/accumulo/test/functional/MergeIT.java | 4 +-
.../accumulo/test/functional/SplitRecoveryIT.java | 3 +-
.../functional/TabletManagementIteratorIT.java | 5 +-
33 files changed, 334 insertions(+), 310 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
index d805b230b2..0bec78d196 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
@@ -35,6 +35,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -48,8 +49,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-import com.google.common.hash.HashCode;
-import com.google.common.hash.Hashing;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -64,9 +63,8 @@ public abstract class AbstractFateStore implements
FateStore {
public static final FateIdGenerator DEFAULT_FATE_ID_GENERATOR = new
FateIdGenerator() {
@Override
public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey
fateKey) {
- HashCode hashCode =
Hashing.murmur3_128().hashBytes(fateKey.getSerialized());
- long tid = hashCode.asLong() & 0x7fffL;
- return FateId.from(instanceType, tid);
+ UUID txUUID = UUID.nameUUIDFromBytes(fateKey.getSerialized());
+ return FateId.from(instanceType, txUUID);
}
};
@@ -271,9 +269,9 @@ public abstract class AbstractFateStore implements
FateStore {
// mean a collision
if (status == TStatus.NEW) {
Preconditions.checkState(tFateKey.isPresent(), "Tx Key is missing from
tid %s",
-fateId.getTid());
+fateId.getTxUUIDStr());
Preconditions.checkState(fateKey.equals(tFateKey.orElseThrow()),
-"Collision detected for tid %s", fateId.getT
(accumulo) branch elasticity updated: Deletes FateTxId (#4370)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new b9867c4c8c Deletes FateTxId (#4370)
b9867c4c8c is described below
commit b9867c4c8cd97b23800b1e129bfa492381ca74c2
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Fri Mar 15 19:08:52 2024 -0400
Deletes FateTxId (#4370)
FateTxId has now been fully replaced by FateId, and can be safely
deleted
---
.../org/apache/accumulo/core/fate/FateTxId.java| 62 --
.../apache/accumulo/core/util/FastFormatTest.java | 13 ++---
2 files changed, 5 insertions(+), 70 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/FateTxId.java
b/core/src/main/java/org/apache/accumulo/core/fate/FateTxId.java
deleted file mode 100644
index ad0d5740af..00
--- a/core/src/main/java/org/apache/accumulo/core/fate/FateTxId.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.core.fate;
-
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.util.FastFormat;
-
-import com.google.common.base.Preconditions;
-
-public class FateTxId {
-
- private static final String PREFIX = "FATE[";
- private static final String SUFFIX = "]";
-
- private final static Pattern PATTERN =
- Pattern.compile(Pattern.quote(PREFIX) + "[0-9a-fA-F]+" +
Pattern.quote(SUFFIX));
-
- private static String getHex(String fmtTid) {
-return fmtTid.substring(PREFIX.length(), fmtTid.length() -
SUFFIX.length());
- }
-
- /**
- * @return true if string was created by {@link #formatTid(long)} and false
otherwise.
- */
- public static boolean isFormatedTid(String fmtTid) {
-return PATTERN.matcher(fmtTid).matches();
- }
-
- /**
- * Reverses {@link #formatTid(long)}
- */
- public static long fromString(String fmtTid) {
-Preconditions.checkArgument(fmtTid.startsWith(PREFIX) &&
fmtTid.endsWith(SUFFIX));
-return Long.parseLong(getHex(fmtTid), 16);
- }
-
- /**
- * Formats transaction ids in a consistent way that is useful for logging
and persisting.
- */
- public static String formatTid(long tid) {
-// do not change how this formats without considering implications for
persistence
-return FastFormat.toHexString(PREFIX, tid, SUFFIX);
- }
-
-}
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
b/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
index 40da603e4a..8e68d87a34 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/FastFormatTest.java
@@ -24,7 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.Arrays;
-import org.apache.accumulo.core.fate.FateTxId;
+import org.apache.accumulo.core.fate.FateId;
import org.junit.jupiter.api.Test;
public class FastFormatTest {
@@ -121,13 +121,10 @@ public class FastFormatTest {
@Test
public void testHexString() {
-final String PREFIX = "FATE[";
-final String SUFFIX = "]";
-String formattedTxId = FateTxId.formatTid(64L);
-String hexStr = FastFormat.toHexString(PREFIX, 64L, SUFFIX);
-assertEquals(formattedTxId, hexStr);
-long txid = FateTxId.fromString("FATE[2e429160071c63d8]");
-assertEquals("FATE[2e429160071c63d8]", FastFormat.toHexString(PREFIX,
txid, SUFFIX));
+String prefix = "FATE:USER:";
+assertEquals(FateId.formatTid(987654321L),
FastFormat.toHexString(987654321L));
+long txid = FateId.from(prefix + "2e429160071c63d8").getTid();
+assertEquals(prefix + "2e429160071c63d8", FastFormat.toHexString(prefix,
txid, ""));
assertEquals(String.format("%016x", 64L), FastFormat.toHexString(64L));
assertEquals(String.format("%016x", 0X2e429160071c63d8L),
FastFormat.toHexString(0X2e429160071c63d8L));
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9550da7c838994f9f6e9547819566b01e4b1dc6e
Merge: ae49a7ff02 9839e4d42f
Author: Keith Turner
AuthorDate: Thu Mar 14 10:29:24 2024 -0400
Merge branch 'main' into elasticity
.../accumulo/core/fate/AbstractFateStore.java | 12 ++-
.../org/apache/accumulo/core/fate/AdminUtil.java | 6 +++---
.../java/org/apache/accumulo/core/fate/Fate.java | 15 +++---
.../org/apache/accumulo/core/fate/FateCleaner.java | 3 ++-
.../org/apache/accumulo/core/fate/FateStore.java | 6 +++---
.../accumulo/core/fate/WrappedFateTxStore.java | 6 +++---
.../apache/accumulo/core/fate/FateCleanerTest.java | 21 ++--
.../org/apache/accumulo/core/fate/TestStore.java | 4 ++--
.../test/compaction/ExternalCompaction_1_IT.java | 6 +++---
.../accumulo/test/fate/accumulo/FateStoreIT.java | 23 +++---
10 files changed, 53 insertions(+), 49 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
index 0cad25f857,00..d805b230b2
mode 100644,00..100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
@@@ -1,494 -1,0 +1,496 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.fate;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
+import java.time.Duration;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
++import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.core.fate.Fate.TxInfo;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.time.NanoTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public abstract class AbstractFateStore implements FateStore {
+
+ private static final Logger log =
LoggerFactory.getLogger(AbstractFateStore.class);
+
+ // Default maximum size of 100,000 transactions before deferral is stopped
and
+ // all existing transactions are processed immediately again
+ public static final int DEFAULT_MAX_DEFERRED = 100_000;
+
+ public static final FateIdGenerator DEFAULT_FATE_ID_GENERATOR = new
FateIdGenerator() {
+@Override
+public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey
fateKey) {
+ HashCode hashCode =
Hashing.murmur3_128().hashBytes(fateKey.getSerialized());
+ long tid = hashCode.asLong() & 0x7fffL;
+ return FateId.from(instanceType, tid);
+}
+ };
+
+ protected final Set reserved;
+ protected final Map deferred;
+ private final int maxDeferred;
+ private final AtomicBoolean deferredOverflow = new AtomicBoolean();
+ private final FateIdGenerator fateIdGenerator;
+
+ // This is incremented each time a transaction was unreserved that was non
new
+ protected final SignalCount unreservedNonNewCount = new SignalCount();
+
+ // This is incremented each time a transaction is unreserved that was
runnable
+ protected final SignalCount unreservedRunnableCount = new SignalCount();
+
+ public AbstractFateStore() {
+this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(int maxDeferred, FateIdG
(accumulo) branch elasticity updated (ae49a7ff02 -> 9550da7c83)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from ae49a7ff02 Merge branch 'main' into elasticity add 23e17129de Revert #4358 - Replace long + TimeUnit with Duration in ReadOnlyTStore.unreserve() add 05c2f45042 Reduced warning logs under normal conditions in compaction coordinator (#4362) add 9839e4d42f Merge branch '2.1' new 9550da7c83 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/fate/AbstractFateStore.java | 12 ++- .../org/apache/accumulo/core/fate/AdminUtil.java | 6 +++--- .../java/org/apache/accumulo/core/fate/Fate.java | 15 +++--- .../org/apache/accumulo/core/fate/FateCleaner.java | 3 ++- .../org/apache/accumulo/core/fate/FateStore.java | 6 +++--- .../accumulo/core/fate/WrappedFateTxStore.java | 6 +++--- .../apache/accumulo/core/fate/FateCleanerTest.java | 21 ++-- .../org/apache/accumulo/core/fate/TestStore.java | 4 ++-- .../test/compaction/ExternalCompaction_1_IT.java | 6 +++--- .../accumulo/test/fate/accumulo/FateStoreIT.java | 23 +++--- 10 files changed, 53 insertions(+), 49 deletions(-)
(accumulo) branch elasticity updated (8384b9be6a -> ae49a7ff02)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8384b9be6a Fate Op Command Updates and Tests (#4350) add 162b8effb1 Replace long + TimeUnit with Duration in ReadOnlyTStore.unreserve() (#4358) add 92331ea113 Throw error when non-standard chars exist (#4348) add 171a1e144c Merge branch '2.1' add ae6085c346 Adds NanoTime wrapper for System.nanoTime (#4364) new ae49a7ff02 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/fate/AbstractFateStore.java | 24 +-- .../org/apache/accumulo/core/fate/AdminUtil.java | 6 +- .../java/org/apache/accumulo/core/fate/Fate.java | 15 +- .../org/apache/accumulo/core/fate/FateCleaner.java | 3 +- .../org/apache/accumulo/core/fate/FateStore.java | 6 +- .../accumulo/core/fate/WrappedFateTxStore.java | 6 +- .../accumulo/core/file/rfile/GenerateSplits.java | 32 +++- .../apache/accumulo/core/util/time/NanoTime.java | 104 + .../apache/accumulo/core/fate/FateCleanerTest.java | 21 ++- .../org/apache/accumulo/core/fate/TestStore.java | 4 +- .../core/file/rfile/GenerateSplitsTest.java| 64 +++- .../accumulo/core/util/time/NanoTimeTest.java | 162 + .../java/org/apache/accumulo/manager/Manager.java | 9 +- .../test/compaction/ExternalCompaction_1_IT.java | 6 +- .../accumulo/test/fate/accumulo/FateStoreIT.java | 23 ++- 15 files changed, 410 insertions(+), 75 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/util/time/NanoTime.java create mode 100644 core/src/test/java/org/apache/accumulo/core/util/time/NanoTimeTest.java
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit ae49a7ff027178a387d39f048e1090bcffe11162
Merge: 8384b9be6a ae6085c346
Author: Keith Turner
AuthorDate: Wed Mar 13 15:12:41 2024 -0400
Merge branch 'main' into elasticity
.../accumulo/core/fate/AbstractFateStore.java | 24 +--
.../org/apache/accumulo/core/fate/AdminUtil.java | 6 +-
.../java/org/apache/accumulo/core/fate/Fate.java | 15 +-
.../org/apache/accumulo/core/fate/FateCleaner.java | 3 +-
.../org/apache/accumulo/core/fate/FateStore.java | 6 +-
.../accumulo/core/fate/WrappedFateTxStore.java | 6 +-
.../accumulo/core/file/rfile/GenerateSplits.java | 32 +++-
.../apache/accumulo/core/util/time/NanoTime.java | 104 +
.../apache/accumulo/core/fate/FateCleanerTest.java | 21 ++-
.../org/apache/accumulo/core/fate/TestStore.java | 4 +-
.../core/file/rfile/GenerateSplitsTest.java| 64 +++-
.../accumulo/core/util/time/NanoTimeTest.java | 162 +
.../java/org/apache/accumulo/manager/Manager.java | 9 +-
.../test/compaction/ExternalCompaction_1_IT.java | 6 +-
.../accumulo/test/fate/accumulo/FateStoreIT.java | 23 ++-
15 files changed, 410 insertions(+), 75 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
index 702313f5ab,00..0cad25f857
mode 100644,00..100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java
@@@ -1,494 -1,0 +1,494 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.fate;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
++import java.time.Duration;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
- import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.core.fate.Fate.TxInfo;
+import org.apache.accumulo.core.util.Pair;
++import org.apache.accumulo.core.util.time.NanoTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public abstract class AbstractFateStore implements FateStore {
+
+ private static final Logger log =
LoggerFactory.getLogger(AbstractFateStore.class);
+
+ // Default maximum size of 100,000 transactions before deferral is stopped
and
+ // all existing transactions are processed immediately again
+ public static final int DEFAULT_MAX_DEFERRED = 100_000;
+
+ public static final FateIdGenerator DEFAULT_FATE_ID_GENERATOR = new
FateIdGenerator() {
+@Override
+public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey
fateKey) {
+ HashCode hashCode =
Hashing.murmur3_128().hashBytes(fateKey.getSerialized());
+ long tid = hashCode.asLong() & 0x7fffL;
+ return FateId.from(instanceType, tid);
+}
+ };
+
+ protected final Set reserved;
- protected final Map deferred;
++ protected final Map deferred;
+ private final int maxDeferred;
+ private final AtomicBoolean deferredOverflow = new AtomicBoolean();
+ private final FateIdGenerator fateIdGenerator;
+
+ // This is incremented each time a transaction was unreserved that was non
new
+ protected final SignalCount unreservedNonNewCount = new SignalCount();
+
+ // This
(accumulo) branch main updated: Adds NanoTime wrapper for System.nanoTime (#4364)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ae6085c346 Adds NanoTime wrapper for System.nanoTime (#4364)
ae6085c346 is described below
commit ae6085c34677e752949ef093cb350281b41a275f
Author: Keith Turner
AuthorDate: Wed Mar 13 14:08:17 2024 -0400
Adds NanoTime wrapper for System.nanoTime (#4364)
* Adds NanoTime wrapper for System.nanoTime
Adds a strong type for System.nanoTime() and uses it in a few places.
Could be used in many more places if this is merged.
Co-authored-by: EdColeman
---
.../org/apache/accumulo/core/fate/ZooStore.java| 15 +-
.../apache/accumulo/core/util/time/NanoTime.java | 104 +
.../accumulo/core/util/time/NanoTimeTest.java | 162 +
.../java/org/apache/accumulo/manager/Manager.java | 9 +-
4 files changed, 280 insertions(+), 10 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
index 941c04c241..c3de5f29df 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
@@ -39,12 +39,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.time.NanoTime;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
@@ -64,7 +64,7 @@ public class ZooStore implements TStore {
private ZooReaderWriter zk;
private String lastReserved = "";
private Set reserved;
- private Map deferred; // use Long here to properly handle
System.nanoTime()
+ private Map deferred;
private long statusChangeEvents = 0;
private int reservationsWaiting = 0;
@@ -164,7 +164,7 @@ public class ZooStore implements TStore {
}
if (deferred.containsKey(tid)) {
- if (deferred.get(tid) - System.nanoTime() < 0) {
+ if (deferred.get(tid).elapsed().compareTo(Duration.ZERO) > 0) {
deferred.remove(tid);
} else {
continue;
@@ -203,10 +203,11 @@ public class ZooStore implements TStore {
if (deferred.isEmpty()) {
this.wait(5000);
} else {
- final long now = System.nanoTime();
- long minWait = deferred.values().stream().mapToLong(l -> l -
now).min().orElseThrow();
+ var now = NanoTime.now();
+ long minWait = deferred.values().stream()
+ .mapToLong(nanoTime ->
nanoTime.subtract(now).toMillis()).min().orElseThrow();
if (minWait > 0) {
-this.wait(Math.min(TimeUnit.NANOSECONDS.toMillis(minWait),
5000));
+this.wait(Math.min(minWait, 5000));
}
}
}
@@ -284,7 +285,7 @@ public class ZooStore implements TStore {
}
if (deferTime.compareTo(Duration.ZERO) > 0) {
-deferred.put(tid, deferTime.toNanos() + System.nanoTime());
+deferred.put(tid, NanoTime.nowPlus(deferTime));
}
this.notifyAll();
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/time/NanoTime.java
b/core/src/main/java/org/apache/accumulo/core/util/time/NanoTime.java
new file mode 100644
index 00..f081278589
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/util/time/NanoTime.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.ut
(accumulo) branch elasticity updated: Fate Op Command Updates and Tests (#4350)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 8384b9be6a Fate Op Command Updates and Tests (#4350)
8384b9be6a is described below
commit 8384b9be6abe786ad855d5457eba6bceb3868025
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Wed Mar 13 14:07:33 2024 -0400
Fate Op Command Updates and Tests (#4350)
Changes:
- Update summary FateOpsCommand
- Prints full FateId not just the long
- Option to filter based on FateInstanceType
- Option to filter based on FateId
- Originally, the summary command printed all Fate
transactions, thought it might be good to optionally allow filtering by certain
Fate transactions like print allows
- Update print FateOpsCommand
- Prints full FateId not just the long
- Option to filter based on FateInstanceType
- Option to filter based on FateId
- Originally, could filter by long id, but with the
replacement of long transaction id by FateId, this had to be updated.
- Instead of receiving "" or "FATE[]" on
cmd line, now expects a FateId
- Update cancel, delete, and fail FateOpsCommand
- Now work with both ZooStore and AccumuloStore by taking the full
FateId on the command line and determining the store based on the
FateInstanceType of the FateId
- Added tests for all 5 FateOpsCommands (FateOpsCommandsIT)
- Tests using both AccumuloStore (AccumuloFateOpsCommandsIT) and
ZooStore (ZookeeperFateOpsCommandsIT)
- Deleted FateSummaryIT, moved the test to FateOpsCommands
* Changes:
- Wait for condition after stopping ServerType.COMPACTOR instead of sleep
- Cleaned up tests to use fewer assertions
- Added attempt to --delete and --fail a transaction when the Manager is
still alive in testFateDeleteCommand() and testFateFailCommand()
---
.../org/apache/accumulo/core/fate/AdminUtil.java | 239 +
.../org/apache/accumulo/core/fate/FateTxId.java| 8 -
.../org/apache/accumulo/server/util/Admin.java | 110 ++--
.../server/util/fateCommand/FateSummaryReport.java | 34 +-
.../server/util/fateCommand/FateTxnDetails.java| 24 +-
.../server/util/fateCommand/SummaryReportTest.java | 11 +-
.../server/util/fateCommand/TxnDetailsTest.java| 7 +-
.../manager/metrics/fate/FateMetricValues.java | 2 +-
.../org/apache/accumulo/test/FateSummaryIT.java| 156 --
.../accumulo/test/fate/FateOpsCommandsIT.java | 564 +
.../fate/accumulo/AccumuloFateOpsCommandsIT.java | 32 ++
.../fate/zookeeper/ZookeeperFateOpsCommandsIT.java | 37 ++
.../test/functional/FateConcurrencyIT.java | 14 +-
.../test/functional/FunctionalTestUtils.java | 2 +-
14 files changed, 888 insertions(+), 352 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
index 2a436b3444..c18defb1ac 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
@@ -46,7 +46,6 @@ import
org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.lock.ServiceLock.ServiceLockPath;
-import org.apache.accumulo.core.util.FastFormat;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,7 +74,7 @@ public class AdminUtil {
*/
public static class TransactionStatus {
-private final long txid;
+private final FateId fateId;
private final FateInstanceType instanceType;
private final TStatus status;
private final String txName;
@@ -84,10 +83,10 @@ public class AdminUtil {
private final String top;
private final long timeCreated;
-private TransactionStatus(Long tid, FateInstanceType instanceType, TStatus
status,
+private TransactionStatus(FateId fateId, FateInstanceType instanceType,
TStatus status,
String txName, List hlocks, List wlocks, String top,
Long timeCreated) {
- this.txid = tid;
+ this.fateId = fateId;
this.instanceType = instanceType;
this.status = status;
this.txName = txName;
@@ -102,8 +101,8 @@ public class AdminUtil {
* @return This fate operations transaction id, formatted in the same way
as FATE transactions
* are in the Accumulo logs.
*/
-public String getTxid() {
- return FastFormat.toHexString(txid);
+public FateId getFateI
(accumulo) branch elasticity updated: partially avoids splitting and compacting offline tablets (#4343)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 744dc5286d partially avoids splitting and compacting offline tablets
(#4343)
744dc5286d is described below
commit 744dc5286ddc92cf8007c46d664e1deef9afe56d
Author: Keith Turner
AuthorDate: Tue Mar 12 11:18:46 2024 -0400
partially avoids splitting and compacting offline tablets (#4343)
Modifies TabletManagementIterator so that it does not return tablets
that need split of compaction of if the table is offline. This
partially prevents those tablets from splitting. Still need to handle
things that were queued for split of compaction when a table is taken
offline. Once those other changes are made this change will still avoid
queuing and processing tablets for work that does not need to be done.
---
.../manager/state/TabletManagementIterator.java| 2 +-
.../functional/TabletManagementIteratorIT.java | 29 +++---
2 files changed, 27 insertions(+), 4 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
index 4704f691c8..2e6627c78e 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
@@ -270,7 +270,7 @@ public class TabletManagementIterator extends
SkippingIterator {
reasonsToReturnThisTablet.add(ManagementAction.NEEDS_LOCATION_UPDATE);
}
-if (tm.getOperationId() == null
+if (tm.getOperationId() == null &&
tabletMgmtParams.isTableOnline(tm.getTableId())
&& Collections.disjoint(REASONS_NOT_TO_SPLIT_OR_COMPACT,
reasonsToReturnThisTablet)) {
try {
if (shouldReturnDueToSplit(tm,
this.env.getPluginEnv().getConfiguration(tm.getTableId( {
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
index c88989d1c4..34723f3fae 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
@@ -50,6 +50,7 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
@@ -112,7 +113,7 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- String[] tables = getUniqueNames(9);
+ String[] tables = getUniqueNames(10);
final String t1 = tables[0];
final String t2 = tables[1];
final String t3 = tables[2];
@@ -122,6 +123,7 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
final String metaCopy3 = tables[6];
final String metaCopy4 = tables[7];
final String metaCopy5 = tables[8];
+ final String metaCopy6 = tables[9];
// create some metadata
createTable(client, t1, true);
@@ -156,6 +158,7 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
copyTable(client, metaCopy1, metaCopy3);
copyTable(client, metaCopy1, metaCopy4);
copyTable(client, metaCopy1, metaCopy5);
+ copyTable(client, metaCopy1, metaCopy6);
// t1 is unassigned, setting to always will generate a change to host
tablets
setTabletAvailability(client, metaCopy1, t1,
TabletAvailability.HOSTED.name());
@@ -240,8 +243,27 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
assertEquals(1, findTabletsNeedingAttention(client, metaCopy4,
tabletMgmtParams),
"Should have one tablet that needs a volume replacement");
+ // In preparation for split an offline testing ensure nothing needs
attention
+ tabletMgmtParams = createParameters(client);
+ addFiles(client, metaCopy6, t4);
+ assertEquals(0, findTabletsNeedingAttention(client, metaCopy6,
tabletMgmtParams),
+ "No tablets should need attention");
+ // Lower the split threshold for the table, should cause the files added
to need attention.
+ client.tableOperations().setProperty(tables[3],
P
(accumulo) branch elasticity updated: Updates merge code to handle all tablet metadata columns (#4352)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 3fe1115014 Updates merge code to handle all tablet metadata columns
(#4352)
3fe1115014 is described below
commit 3fe11150145fab6d920aae8582340ad7e07a8fba
Author: Keith Turner
AuthorDate: Fri Mar 8 19:59:27 2024 -0500
Updates merge code to handle all tablet metadata columns (#4352)
Adds tests for all tablet metadata columns to ensure the merge code
handles them. Some columns were not handled, so new code was added to
handle them.
In the process of adding unit test for the merge code, also added test
for some odd conditions in the code that would be hard to test in an
integration test.
---
.../manager/tableOps/merge/DeleteRows.java | 11 +-
.../manager/tableOps/merge/MergeTablets.java | 70 +++-
.../manager/tableOps/merge/MergeTabletsTest.java | 466 +
.../manager/tableOps/split/UpdateTabletsTest.java | 2 +-
4 files changed, 527 insertions(+), 22 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
index f071785ece..2c5221b7ce 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java
@@ -18,11 +18,6 @@
*/
package org.apache.accumulo.manager.tableOps.merge;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static
org.apache.accumulo.manager.tableOps.merge.MergeTablets.validateTablet;
import java.util.ArrayList;
@@ -88,9 +83,9 @@ public class DeleteRows extends ManagerRepo {
var opid = TabletOperationId.from(TabletOperationType.MERGING, fateId);
try (
-var tabletsMetadata = manager.getContext().getAmple().readTablets()
-.forTable(range.tableId()).overlapping(range.prevEndRow(),
range.endRow())
-.fetch(OPID, LOCATION, FILES, PREV_ROW,
LOGS).checkConsistency().build();
+var tabletsMetadata =
+
manager.getContext().getAmple().readTablets().forTable(range.tableId())
+.overlapping(range.prevEndRow(),
range.endRow()).checkConsistency().build();
var tabletsMutator =
manager.getContext().getAmple().conditionallyMutateTablets()) {
KeyExtent firstCompleteContained = null;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
index d48528a3f7..50703849c6 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
@@ -18,16 +18,6 @@
*/
package org.apache.accumulo.manager.tableOps.merge;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
import static
org.apache.accumulo.manager.tableOps.merge.DeleteRows.verifyAccepted;
import java.util.ArrayList;
@@ -87,12 +77,12 @@ public class MergeTablets extends ManagerRepo {
TabletMetadata lastTabletMeta = null;
try (var tabletsMetadata = manager.getContext().getAmple().readTablets()
-.forTable(range.tableId()).overlapping(range.prevEndRow(),
range.endRow())
-.fetch(OPID, LOCATION, AVAILABILITY, FILES, TIME, DIR, ECOMP,
PREV_ROW, LOGS, MERGED)
-.build
(accumulo) branch elasticity updated: Stop waiting on fate when reporting compaction complete (#4339)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5c5150242d Stop waiting on fate when reporting compaction complete
(#4339)
5c5150242d is described below
commit 5c5150242d160160bfcf8aed3cfe6fdcefd81da5
Author: Keith Turner
AuthorDate: Fri Mar 8 19:58:45 2024 -0500
Stop waiting on fate when reporting compaction complete (#4339)
Compactors were waiting on the Fate transaction that does compaction
commit. They were doing this so that the dead compaction detector
would not kill the compaction during commit. This change removes
the need for compactors to wait on compaction commit, freeing them
to run another compaction.
To accomplish this the dead compaction detector was modified to look
for fate operations commiting compactions. This work leveraged the
new FateKey. A bonus for this changes is that duplicate RPC messages
reporting a compaction as completed will no longer spin up duplicate
Fate operations to commit. The Fate operations would have likely
been harmless, but would have consumed resources.
A new method was added to Fate to list transactions with a certain
FateKeyType. Tests were added for this new Fate method.
An integration test was added to ensure the dead compaction detector
does not delete compactions that no compactors knows of but do have
a Fate transaction committing them.
---
.../java/org/apache/accumulo/core/fate/Fate.java | 8 ++
.../accumulo/core/fate/ReadOnlyFateStore.java | 5 ++
.../org/apache/accumulo/core/fate/ZooStore.java| 6 ++
.../accumulo/core/fate/accumulo/AccumuloStore.java | 14
.../apache/accumulo/core/logging/FateLogger.java | 5 ++
.../org/apache/accumulo/core/fate/TestStore.java | 5 ++
.../coordinator/CompactionCoordinator.java | 33 ++--
.../coordinator/DeadCompactionDetector.java| 97 --
.../coordinator/commit/PutGcCandidates.java| 11 +--
.../coordinator/commit/RefreshTablet.java | 7 +-
.../test/compaction/ExternalCompaction_1_IT.java | 97 ++
.../accumulo/test/fate/accumulo/FateStoreIT.java | 65 +++
12 files changed, 318 insertions(+), 35 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
b/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
index 0b82f73f11..e5be68dbb2 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
@@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TransferQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
+import java.util.stream.Stream;
import
org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -482,6 +483,13 @@ public class Fate {
}
}
+ /**
+ * Lists transctions for a given fate key type.
+ */
+ public Stream list(FateKey.FateKeyType type) {
+return store.list(type);
+ }
+
/**
* Initiates shutdown of background threads and optionally waits on them.
*/
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/ReadOnlyFateStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/ReadOnlyFateStore.java
index bdbb7739f9..b2aa4999b2 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/ReadOnlyFateStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/ReadOnlyFateStore.java
@@ -138,6 +138,11 @@ public interface ReadOnlyFateStore {
*/
Stream list();
+ /**
+ * list transaction in the store that have a given fate key type.
+ */
+ Stream list(FateKey.FateKeyType type);
+
/**
* Finds all fate ops that are (IN_PROGRESS, SUBMITTED, or
FAILED_IN_PROGRESS) and unreserved. Ids
* that are found are passed to the consumer. This method will block until
at least one runnable
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
index 6813e727c5..af6fd233de 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
@@ -377,6 +377,12 @@ public class ZooStore extends AbstractFateStore {
}
}
+ @Override
+ public Stream list(FateKey.FateKeyType type) {
+return getTransactions().flatMap(fis -> getKey(fis.getFateId()).stream())
+.filter(fateKey -> fateKey.getType() == type);
+ }
+
protected static class NodeValue {
final TStatus status;
final Optional fateKey;
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/accumulo/AccumuloStore.j
(accumulo) branch elasticity updated: fixes bug and removes unneeded wait in delete table (#4346)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c7ae69b881 fixes bug and removes unneeded wait in delete table (#4346)
c7ae69b881 is described below
commit c7ae69b881da00113f2ceef642f5f470818d569d
Author: Keith Turner
AuthorDate: Thu Mar 7 13:31:48 2024 -0500
fixes bug and removes unneeded wait in delete table (#4346)
Noticed an integration test was taking 0 to 5 seconds to delete a table.
Investigated this and found some code in delete table that was waiting
on the TabletGroupWatcher to cycle as the cause of the delay.
Investigated why this wait was there and could not find a definite
answer. Speculating that the purpose of the wait was to deal with
possible concurrent writes to tablet metadata by the tablet group
watcher in the case it has not yet noticed the table state is DELETING.
Changes in elasticity make this wait unnecessary because the delete table
fate sets an operation id on each tablet that should prevent any writes
to the tablet after its set. After being set this would prevent any
in flight writes by the tablet group watcher from succeeding. Before
elasticity there was no way for the delete table fate to prevent writes
by the tablet group watcher, so thinking it just waited until it was
certain the tablet group watcher was aware of the new table state and
would therefore do no writes.
While researching I noticed the delete table code in 2.1 logged per
tablet information about any tablet its was waiting on. This was not
being done in elasticity, so added logging for this.
With these changes TabletManagementIteratorIT went from 1 min runtime to
30 second runtime. The test deleted 9 tables.
---
.../java/org/apache/accumulo/manager/Manager.java | 10 --
.../apache/accumulo/manager/state/TableStats.java | 4
.../accumulo/manager/tableOps/delete/CleanUp.java | 23 --
.../manager/tableOps/delete/DeleteTable.java | 2 +-
.../manager/tableOps/delete/ReserveTablets.java| 13 +++-
5 files changed, 13 insertions(+), 39 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 935ca7d262..95fb00777e 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -520,16 +520,6 @@ public class Manager extends AbstractServer
}
}
- public boolean hasCycled(long time) {
-for (TabletGroupWatcher watcher : watchers) {
- if (watcher.stats.lastScanFinished() < time) {
-return false;
- }
-}
-
-return true;
- }
-
public void clearMigrations(TableId tableId) {
synchronized (migrations) {
migrations.keySet().removeIf(extent -> extent.tableId().equals(tableId));
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
index 3e2380a57c..c0d9a4f5e3 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/state/TableStats.java
@@ -76,10 +76,6 @@ public class TableStats {
return endScan - startScan;
}
- public synchronized long lastScanFinished() {
-return endScan;
- }
-
@Override
public String toString() {
return new StringBuilder().append("last:
").append(last.toString()).toString();
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
index 9d8d2b86f6..9dd234eba9 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java
@@ -61,32 +61,9 @@ class CleanUp extends ManagerRepo {
private long creationTime;
- private void readObject(java.io.ObjectInputStream in) throws IOException,
ClassNotFoundException {
-in.defaultReadObject();
-
-// handle the case where we start executing on a new machine where the
current time is in the
-// past relative to the previous machine
-// if the new machine has time in the future, that will work ok w/
hasCycled
-if (System.currentTimeMillis() < creationTime) {
- creationTime = System.currentTimeMillis();
-}
-
- }
-
public CleanUp(TableId tableId, NamespaceId namespaceId) {
this.tableId = tableId;
this.namespaceId = namespaceId;
-creationTime = Syste
(accumulo) branch elasticity updated: Updates compaction to use TabletLogger (#4333)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 202198a588 Updates compaction to use TabletLogger (#4333)
202198a588 is described below
commit 202198a588aae1096611ff9b5513075054e2a876
Author: Keith Turner
AuthorDate: Mon Mar 4 10:55:50 2024 -0500
Updates compaction to use TabletLogger (#4333)
---
.../apache/accumulo/core/logging/TabletLogger.java | 49 +++---
.../coordinator/CompactionCoordinator.java | 9 ++--
.../coordinator/commit/CommitCompaction.java | 4 ++
.../manager/tableOps/compact/CompactionDriver.java | 22 --
test/src/main/resources/log4j2-test.properties | 3 ++
5 files changed, 55 insertions(+), 32 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
index 349d29b19f..e76c62a6c9 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
@@ -27,14 +27,16 @@ import java.util.SortedSet;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
@@ -127,40 +129,36 @@ public class TabletLogger {
cf -> CompactableFileImpl.toStoredTabletFile(cf).toMinimalString());
}
- public static void selected(KeyExtent extent, CompactionKind kind,
+ public static void selected(FateId fateId, KeyExtent extent,
Collection inputs) {
-fileLog.trace("{} changed compaction selection set for {} new set {}",
extent, kind,
+fileLog.trace("Selected files {} {} {}", extent, fateId,
Collections2.transform(inputs, StoredTabletFile::toMinimalString));
}
- public static void compacting(KeyExtent extent, CompactionJob job,
CompactionConfig config) {
+ public static void compacting(TabletMetadata tabletMetadata,
ExternalCompactionId cid,
+ String compactorAddress, CompactionJob job) {
if (fileLog.isDebugEnabled()) {
- if (config == null) {
-fileLog.debug("Compacting {} on {} for {} from {} size {}", extent,
job.getGroup(),
-job.getKind(), asMinimalString(job.getFiles()),
getSize(job.getFiles()));
+ if (job.getKind() == CompactionKind.USER) {
+var fateId = tabletMetadata.getSelectedFiles().getFateId();
+fileLog.debug(
+"Compacting {} driver:{} id:{} group:{} compactor:{} priority:{}
size:{} kind:{} files:{}",
+tabletMetadata.getExtent(), fateId, cid, job.getGroup(),
compactorAddress,
+job.getPriority(), getSize(job.getFiles()), job.getKind(),
+asMinimalString(job.getFiles()));
} else {
-fileLog.debug("Compacting {} on {} for {} from {} size {} config {}",
extent,
-job.getGroup(), job.getKind(), asMinimalString(job.getFiles()),
getSize(job.getFiles()),
-config);
+fileLog.debug(
+"Compacting {} id:{} group:{} compactor:{} priority:{} size:{}
kind:{} files:{}",
+tabletMetadata.getExtent(), cid, job.getGroup(), compactorAddress,
job.getPriority(),
+getSize(job.getFiles()), job.getKind(),
asMinimalString(job.getFiles()));
}
}
}
- public static void compacted(KeyExtent extent, CompactionJob job,
StoredTabletFile output) {
-fileLog.debug("Compacted {} for {} created {} from {}", extent,
job.getKind(), output,
-asMinimalString(job.getFiles()));
- }
-
- public static void compactionFailed(KeyExtent extent, CompactionJob job,
- CompactionConfig config) {
-fileLog.debug("Failed to compact: extent: {}, input files: {}, iterators:
{}", extent,
-asMinimalString(job.getFiles()), config.getIterators());
- }
-
- public static void externalCompactionFailed(KeyExtent extent,
ExternalCompactionId id,
- CompactionJob job, CompactionConf
(accumulo) branch elasticity updated: Considers all tablet metadata columns in split code (#4323)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 1d7f6937fc Considers all tablet metadata columns in split code (#4323)
1d7f6937fc is described below
commit 1d7f6937fcd4a122978486c10832b37c62effea1
Author: Keith Turner
AuthorDate: Mon Mar 4 10:31:48 2024 -0500
Considers all tablet metadata columns in split code (#4323)
Made the following changes to the split code that adds new tablets and
updates the existing tablet.
* fixed potential NPE w/ tablet operation id check by reversing order
of equals check
* Throws IllegalStateException when attempting to split tablet with
merged or cloned markers
* Removed adding wals when creating new tablets in split, its not
expected that the parent tablet would have wals and this is checked
earlier
* Deleted any user compaction requested, hosting requested, suspended,
or last columns in the parent tablet
Added a unit test that attempts to exercise the split code with all
tablet columns. The unit test also has a set of tablet columns that
were verified to work with split and it is checked against the set of
columns in the code. The purpose of this test is to fail when a new
column is added to ensure that split is considered.
Was a bit uncertain about deleting the last location and suspend. Those
columns either need to be deleted from the parent tablet or added to the
new tablets being created. The current code was doing neither. Decided
to delete them as the new tablets have a different range and are
conceptually different tablets than the parent.
---
.../manager/tableOps/split/UpdateTablets.java | 42 ++-
.../manager/tableOps/split/UpdateTabletsTest.java | 328 +
.../apache/accumulo/test/functional/SplitIT.java | 54
3 files changed, 421 insertions(+), 3 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
index 2fe3c56399..f2d1501ae5 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
@@ -82,7 +83,7 @@ public class UpdateTablets extends ManagerRepo {
}
}
-Preconditions.checkState(tabletMetadata.getOperationId().equals(opid),
+Preconditions.checkState(opid.equals(tabletMetadata.getOperationId()),
"Tablet %s does not have expected operation id %s it has %s",
splitInfo.getOriginal(), opid,
tabletMetadata.getOperationId());
@@ -94,6 +95,13 @@ public class UpdateTablets extends ManagerRepo {
"Tablet unexpectedly had walogs %s %s %s", fateId,
tabletMetadata.getLogs(),
tabletMetadata.getExtent());
+Preconditions.checkState(!tabletMetadata.hasMerged(),
+"Tablet unexpectedly has a merged marker %s %s", fateId,
tabletMetadata.getExtent());
+
+Preconditions.checkState(tabletMetadata.getCloned() == null,
+"Tablet unexpectedly has a cloned marker %s %s %s", fateId,
tabletMetadata.getCloned(),
+tabletMetadata.getExtent());
+
var newTablets = splitInfo.getTablets();
var newTabletsFiles = getNewTabletFiles(newTablets, tabletMetadata,
@@ -120,7 +128,7 @@ public class UpdateTablets extends ManagerRepo {
newTablets.forEach(extent -> tabletsFiles.put(extent, new HashMap<>()));
-// determine while files overlap which tablets and their estimated sizes
+// determine which files overlap which tablets and their estimated sizes
tabletMetadata.getFilesMap().forEach((file, dataFileValue) -> {
FileUtil.FileInfo fileInfo = fileInfoProvider.apply(file);
@@ -187,6 +195,7 @@ public class UpdateTablets extends ManagerRepo {
mutator.putTime(tabletMetadata.getTime());
tabletMetadata.getFlushId().ifPresent(mutator::putFlushId);
mutator.putPrevEndRow(newExtent.prevEndRow());
+
tabletMetadata.getCompacted().forEach(mutator::putCompacted);
tabletMetadata.getCompacted().forEach(compactedFateId -> log
@@ -195,7 +204,6 @@ public class UpdateTablets extends ManagerRepo {
mutator.putTabletAvailability(tabletMetadata.getTabletAvailability());
tabletMetadata.getLoaded().forEach((k, v) ->
mutator.putBulkFile(k.getTabletFile(), v));
(accumulo) branch elasticity updated: Unit test compaction reservation and deny offline (#4334)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 01b0ff9356 Unit test compaction reservation and deny offline (#4334)
01b0ff9356 is described below
commit 01b0ff9356b851d8f47368989407fa610441c23d
Author: Keith Turner
AuthorDate: Mon Mar 4 10:30:10 2024 -0500
Unit test compaction reservation and deny offline (#4334)
Changes the compaction coordinator to not return jobs for a table that
is offline. The point where this check was added to the coordinator
needed unit test, so also added the unit test.
---
.../coordinator/CompactionCoordinator.java | 18 ++-
.../compaction/CompactionCoordinatorTest.java | 148 +
2 files changed, 160 insertions(+), 6 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index ebd5d8f764..a07ff50bc4 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -74,6 +74,7 @@ import org.apache.accumulo.core.fate.FateInstanceType;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.iterators.user.HasExternalCompactionsFilter;
import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil;
+import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
@@ -124,6 +125,7 @@ import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.github.benmanes.caffeine.cache.Weigher;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import com.google.common.net.HostAndPort;
@@ -382,9 +384,9 @@ public class CompactionCoordinator
}
- // ELASTICITY_TODO unit test this code
- private boolean canReserveCompaction(TabletMetadata tablet, CompactionJob
job,
- Set jobFiles) {
+ @VisibleForTesting
+ public static boolean canReserveCompaction(TabletMetadata tablet,
CompactionKind kind,
+ Set jobFiles, ServerContext ctx) {
if (tablet == null) {
// the tablet no longer exist
@@ -395,6 +397,10 @@ public class CompactionCoordinator
return false;
}
+if (ctx.getTableState(tablet.getTableId()) != TableState.ONLINE) {
+ return false;
+}
+
if (!tablet.getFiles().containsAll(jobFiles)) {
return false;
}
@@ -406,7 +412,7 @@ public class CompactionCoordinator
return false;
}
-switch (job.getKind()) {
+switch (kind) {
case SYSTEM:
var userRequestedCompactions =
tablet.getUserCompactionsRequested().size();
if (userRequestedCompactions > 0) {
@@ -427,7 +433,7 @@ public class CompactionCoordinator
}
break;
default:
-throw new UnsupportedOperationException("Not currently handling " +
job.getKind());
+throw new UnsupportedOperationException("Not currently handling " +
kind);
}
return true;
@@ -508,7 +514,7 @@ public class CompactionCoordinator
try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) {
var extent = metaJob.getTabletMetadata().getExtent();
-if (!canReserveCompaction(tabletMetadata, metaJob.getJob(), jobFiles))
{
+if (!canReserveCompaction(tabletMetadata, metaJob.getJob().getKind(),
jobFiles, ctx)) {
return null;
}
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
index 770405863e..4c0b2b1d52 100644
---
a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
@@ -18,10 +18,18 @@
*/
package org.apache.accumulo.manager.compaction;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.S
(accumulo) branch elasticity updated: updates IdleProcessMetricsIT to verify config instead of set it. (#4331)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 8798fccd5c updates IdleProcessMetricsIT to verify config instead of
set it. (#4331)
8798fccd5c is described below
commit 8798fccd5c32874cfd2d8f255781820280534e0f
Author: Keith Turner
AuthorDate: Fri Mar 1 13:42:50 2024 -0500
updates IdleProcessMetricsIT to verify config instead of set it. (#4331)
IdleProcessMetricsIT was setting compactor config to the same as the
default. Changed the test to verify the config instead of setting it.
---
.../apache/accumulo/test/functional/IdleProcessMetricsIT.java | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
index c2d2f649ae..36d16b8d2d 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
@@ -18,11 +18,14 @@
*/
package org.apache.accumulo.test.functional;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
@@ -42,10 +45,10 @@ public class IdleProcessMetricsIT extends
SharedMiniClusterBase {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
coreSite) {
- // Configure all compaction planners to use the default resource group so
- // that only 1 compactor is started by MiniAccumuloCluster
- cfg.setProperty(Property.COMPACTION_SERVICE_DEFAULT_GROUPS.getKey(),
- "[{'group':'default'}]".replaceAll("'", "\""));
+ // Verify expectations about the default config. Want to ensure there no
other resource groups
+ // configured.
+ assertEquals(Map.of(Constants.DEFAULT_COMPACTION_SERVICE_NAME, 1),
+ cfg.getClusterServerConfiguration().getCompactorConfiguration());
// Disable the default scan servers and compactors, just start 1
// tablet server in the default group to host the root and metadata
(accumulo) branch elasticity updated: update IdleProcessMetricsIT to work w/ recent changes (#4326)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5bec92248c update IdleProcessMetricsIT to work w/ recent changes
(#4326)
5bec92248c is described below
commit 5bec92248ce3c980681dd5925a207eb9b866df3e
Author: Keith Turner
AuthorDate: Thu Feb 29 18:40:37 2024 -0500
update IdleProcessMetricsIT to work w/ recent changes (#4326)
---
.../org/apache/accumulo/test/functional/IdleProcessMetricsIT.java | 6 +-
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
index 169d54a3c2..c2d2f649ae 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/IdleProcessMetricsIT.java
@@ -44,11 +44,7 @@ public class IdleProcessMetricsIT extends
SharedMiniClusterBase {
// Configure all compaction planners to use the default resource group so
// that only 1 compactor is started by MiniAccumuloCluster
- cfg.setProperty("compaction.service.root.planner.opts.groups",
- "[{'group':'default'}]".replaceAll("'", "\""));
- cfg.setProperty("compaction.service.meta.planner.opts.groups",
- "[{'group':'default'}]".replaceAll("'", "\""));
- cfg.setProperty("compaction.service.default.planner.opts.groups",
+ cfg.setProperty(Property.COMPACTION_SERVICE_DEFAULT_GROUPS.getKey(),
"[{'group':'default'}]".replaceAll("'", "\""));
// Disable the default scan servers and compactors, just start 1
(accumulo) branch elasticity updated: sets initial fate table config (#4320)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new a6f1073ac9 sets initial fate table config (#4320)
a6f1073ac9 is described below
commit a6f1073ac9b95a4a36508e018ac6f24fa499c37f
Author: Keith Turner
AuthorDate: Wed Feb 28 16:32:19 2024 -0500
sets initial fate table config (#4320)
---
.../accumulo/server/init/InitialConfiguration.java | 44 +++---
1 file changed, 23 insertions(+), 21 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
b/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
index e932e0af15..81f2876db5 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
@@ -49,26 +49,30 @@ class InitialConfiguration {
this.hadoopConf = hadoopConf;
this.siteConf = siteConf;
-
initialRootMetaConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(),
"32K");
-initialRootMetaConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
-initialRootMetaConf.put(Property.TABLE_DURABILITY.getKey(), "sync");
-initialRootMetaConf.put(Property.TABLE_MAJC_RATIO.getKey(), "1");
-initialRootMetaConf.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "64M");
-initialRootMetaConf.put(Property.TABLE_CONSTRAINT_PREFIX.getKey() + "1",
-MetadataConstraints.class.getName());
-initialRootMetaConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"scan.vers",
+// config common to all Accumulo tables
+Map commonConfig = new HashMap<>();
+commonConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(),
"32K");
+commonConfig.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
+commonConfig.put(Property.TABLE_DURABILITY.getKey(), "sync");
+commonConfig.put(Property.TABLE_MAJC_RATIO.getKey(), "1");
+commonConfig.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers",
"10," + VersioningIterator.class.getName());
-initialRootMetaConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"scan.vers.opt.maxVersions",
-"1");
-initialRootMetaConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"minc.vers",
+commonConfig.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"scan.vers.opt.maxVersions", "1");
+commonConfig.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.vers",
"10," + VersioningIterator.class.getName());
-initialRootMetaConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"minc.vers.opt.maxVersions",
-"1");
-initialRootMetaConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.vers",
+commonConfig.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"minc.vers.opt.maxVersions", "1");
+commonConfig.put(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers",
"10," + VersioningIterator.class.getName());
-initialRootMetaConf.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.vers.opt.maxVersions",
-"1");
-initialRootMetaConf.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
+commonConfig.put(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.vers.opt.maxVersions", "1");
+commonConfig.put(Property.TABLE_FAILURES_IGNORE.getKey(), "false");
+commonConfig.put(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey(), "");
+commonConfig.put(Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
+commonConfig.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
+
+initialRootMetaConf.putAll(commonConfig);
+initialRootMetaConf.put(Property.TABLE_SPLIT_THRESHOLD.getKey(), "64M");
+initialRootMetaConf.put(Property.TABLE_CONSTRAINT_PREFIX.getKey() + "1",
+MetadataConstraints.class.getName());
initialRootMetaConf.put(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"tablet",
String.format("%s,%s",
MetadataSchema.TabletsSection.TabletColumnFamily.NAME,
MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME));
@@ -78,11 +82,9 @@ class InitialConfiguration {
MetadataSchema.TabletsSection.ServerColumnFamily.NAME,
MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME));
initialRootMetaConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(),
"tablet,server");
-
initialRootMetaConf.put(Property.TABLE_DEFAULT_SCANTIME_V
(accumulo) branch elasticity updated: Added check for tablets with errors, when looking for volume replacements. (#4287)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new cca3683e5b Added check for tablets with errors, when looking for
volume replacements. (#4287)
cca3683e5b is described below
commit cca3683e5b51f66787569e9b968b4599848e136e
Author: Arbaaz Khan
AuthorDate: Tue Feb 27 17:18:36 2024 -0500
Added check for tablets with errors, when looking for volume replacements.
(#4287)
Added check for tablets with errors, when looking for volume replacements.
fixes #4234
---
.../org/apache/accumulo/manager/TabletGroupWatcher.java| 14 +-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index e8bdd794a4..e7ea20413a 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -353,6 +353,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
int[] counts = new int[TabletState.values().length];
private int totalUnloaded;
private long totalVolumeReplacements;
+private int tabletsWithErrors;
}
private TableMgmtStats manageTablets(Iterator iter,
@@ -398,6 +399,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
"Error on TabletServer trying to get Tablet management information
for extent: {}. Error message: {}",
tm.getExtent(), mtiError);
this.metrics.incrementTabletGroupWatcherError(this.store.getLevel());
+tableMgmtStats.tabletsWithErrors++;
continue;
}
@@ -670,7 +672,17 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
iter = store.iterator(tableMgmtParams);
var tabletMgmtStats = manageTablets(iter, tableMgmtParams,
currentTServers, true);
-lookForTabletsNeedingVolReplacement =
tabletMgmtStats.totalVolumeReplacements != 0;
+
+// If currently looking for volume replacements, determine if the next
round needs to look.
+if (lookForTabletsNeedingVolReplacement) {
+ // Continue to look for tablets needing volume replacement if there
was an error
+ // processing tablets in the call to manageTablets() or if we are
still performing volume
+ // replacement. We only want to stop looking for tablets that need
volume replacement when
+ // we have successfully processed all tablet metadata and no more
volume replacements are
+ // being performed.
+ lookForTabletsNeedingVolReplacement =
tabletMgmtStats.totalVolumeReplacements != 0
+ || tabletMgmtStats.tabletsWithErrors != 0;
+}
// provide stats after flushing changes to avoid race conditions w/
delete table
stats.end(managerState);
