[accumulo] branch main updated: Adds unit test for compaction file tracking (#2289)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a8c27d5 Adds unit test for compaction file tracking (#2289)
a8c27d5 is described below
commit a8c27d5286f43abc246e799b60d0ba1e1b3910f9
Author: Keith Turner
AuthorDate: Wed Sep 29 14:13:12 2021 -0400
Adds unit test for compaction file tracking (#2289)
---
.../accumulo/tserver/tablet/CompactableImpl.java | 53 ++-
.../tablet/CompactableImplFileManagerTest.java | 500 +
2 files changed, 534 insertions(+), 19 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 5498ec9..228517e 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -22,6 +22,7 @@ import static
org.apache.accumulo.tserver.TabletStatsKeeper.Operation.MAJOR;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -179,11 +180,11 @@ public class CompactableImpl implements Compactable {
* in a mutually exclusive manner, so synchronization at this level is
unnecessary.
*
*/
- private class FileManager {
+ static abstract class FileManager {
FileSelectionStatus selectStatus = FileSelectionStatus.NOT_ACTIVE;
-private long selectedTime;
-private long selectedExpirationDurationMs;
+private long selectedTimeNanos;
+private Duration selectedExpirationDuration;
private CompactionKind selectKind = null;
// Tracks if when a set of files was selected, if at that time the set was
all of the tablets
@@ -193,7 +194,7 @@ public class CompactableImpl implements Compactable {
private boolean initiallySelectedAll = false;
private Set selectedFiles = new HashSet<>();
-private Set allCompactingFiles = new HashSet<>();
+protected Set allCompactingFiles = new HashSet<>();
// track files produced by compactions of this tablet, those are
considered chopped
private Set choppedFiles = new HashSet<>();
@@ -201,10 +202,10 @@ public class CompactableImpl implements Compactable {
private Set allFilesWhenChopStarted = new HashSet<>();
private final KeyExtent extent;
-private Deriver selectionExpirationDeriver;
+private Deriver selectionExpirationDeriver;
public FileManager(KeyExtent extent, Collection
extCompactingFiles,
-Optional extSelInfo, Deriver
selectionExpirationDeriver) {
+Optional extSelInfo, Deriver
selectionExpirationDeriver) {
this.extent = extent;
this.selectionExpirationDeriver = selectionExpirationDeriver;
@@ -233,8 +234,14 @@ public class CompactableImpl implements Compactable {
return new SelectedInfo(initiallySelectedAll, selectedFiles, selectKind);
}
+protected abstract boolean noneRunning(CompactionKind kind);
+
+protected abstract long getNanoTime();
+
boolean initiateSelection(CompactionKind kind) {
+ Preconditions.checkArgument(kind == CompactionKind.SELECTOR || kind ==
CompactionKind.USER);
+
if (selectStatus == FileSelectionStatus.NOT_ACTIVE || (kind ==
CompactionKind.USER
&& selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR))) {
selectStatus = FileSelectionStatus.NEW;
@@ -262,9 +269,9 @@ public class CompactableImpl implements Compactable {
Preconditions.checkArgument(!selected.isEmpty());
Preconditions.checkState(selectStatus == FileSelectionStatus.SELECTING);
selectStatus = FileSelectionStatus.SELECTED;
- selectedTime = System.currentTimeMillis();
+ selectedTimeNanos = getNanoTime();
// take a snapshot of this from config and use it for the entire
selection for consistency
- selectedExpirationDurationMs = selectionExpirationDeriver.derive();
+ selectedExpirationDuration = selectionExpirationDeriver.derive();
selectedFiles.clear();
selectedFiles.addAll(selected);
initiallySelectedAll = allSelected;
@@ -396,7 +403,7 @@ public class CompactableImpl implements Compactable {
case SELECTED: {
Set candidates = new HashSet<>(currFiles);
candidates.removeAll(allCompactingFiles);
- if (System.currentTimeMillis() - selectedTime <
selectedExpirationDurationMs) {
+ if (getNanoTime() - selectedTimeNanos <
selectedExpirationDuration.toNanos()) {
candidates.removeAll(selectedFiles);
}
return Collections.unmodifiableSet(candidates);
@@ -467,12 +474,1
[accumulo-testing] branch main updated (86536bb -> 27476d0)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from 86536bb Add units to performance test results (#156) add 5561b7c Add documentation, comments, and warning fixes new 27476d0 Update readme and add comments to RandomCachedLookupsPT The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: README.md| 3 +++ .../testing/performance/tests/RandomCachedLookupsPT.java | 9 ++--- 2 files changed, 9 insertions(+), 3 deletions(-)
[accumulo-testing] 01/01: Update readme and add comments to RandomCachedLookupsPT
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit 27476d0fb855ecaae7a04a0343d0e065434448d1 Merge: 86536bb 5561b7c Author: Jeffrey Manno AuthorDate: Wed Sep 29 09:08:20 2021 -0400 Update readme and add comments to RandomCachedLookupsPT * These additions aim to inform the user that some tests may require changes to certain property values in order to run. README.md| 3 +++ .../testing/performance/tests/RandomCachedLookupsPT.java | 9 ++--- 2 files changed, 9 insertions(+), 3 deletions(-)
[accumulo] branch main updated: Upgrade Thrift to 0.15.0 (#2273)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 04bf24f Upgrade Thrift to 0.15.0 (#2273)
04bf24f is described below
commit 04bf24f777a3422d968240c66adf9167ffc4b2e0
Author: Mike Miller
AuthorDate: Wed Sep 29 07:27:37 2021 -0400
Upgrade Thrift to 0.15.0 (#2273)
---
.../core/clientImpl/ThriftTransportPool.java | 26 +++
.../iteratorsImpl/system/SystemIteratorUtil.java | 5 ++-
.../apache/accumulo/core/rpc/FilterTransport.java | 16 +
.../apache/accumulo/core/rpc/TBufferedSocket.java | 3 +-
.../accumulo/core/rpc/TTimeoutTransport.java | 38 --
.../org/apache/accumulo/core/rpc/ThriftUtil.java | 12 +++
.../core/rpc/UGIAssumingTransportFactory.java | 9 -
.../accumulo/core/util/ThriftMessageUtil.java | 21 ++--
core/src/main/scripts/generate-thrift.sh | 2 +-
.../accumulo/core/rpc/TTimeoutTransportTest.java | 9 ++---
.../accumulo/core/util/ThriftMessageUtilTest.java | 2 +-
pom.xml| 2 +-
.../server/rpc/CustomNonBlockingServer.java| 6 ++--
.../org/apache/accumulo/tracer/TraceFormatter.java | 8 ++---
.../org/apache/accumulo/tracer/TraceServer.java| 16 +
.../java/org/apache/accumulo/test/rpc/Mocket.java | 24 +++---
16 files changed, 143 insertions(+), 56 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
index df43f51..4be3e67 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
@@ -42,6 +42,7 @@ import org.apache.accumulo.core.singletons.SingletonService;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.threads.Threads;
+import org.apache.thrift.TConfiguration;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
@@ -654,6 +655,31 @@ public class ThriftTransportPool {
}
}
+@Override
+public TConfiguration getConfiguration() {
+ return wrappedTransport.getConfiguration();
+}
+
+@Override
+public void updateKnownMessageSize(long size) throws TTransportException {
+ try {
+ioCount++;
+wrappedTransport.updateKnownMessageSize(size);
+ } finally {
+ioCount++;
+ }
+}
+
+@Override
+public void checkReadBytesAvailable(long numBytes) throws
TTransportException {
+ try {
+ioCount++;
+wrappedTransport.checkReadBytesAvailable(numBytes);
+ } finally {
+ioCount++;
+ }
+}
+
public ThriftTransportKey getCacheKey() {
return cacheKey;
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/SystemIteratorUtil.java
b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/SystemIteratorUtil.java
index 9e241c2..41a7acc 100644
---
a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/SystemIteratorUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/SystemIteratorUtil.java
@@ -72,9 +72,8 @@ public class SystemIteratorUtil {
}
public static byte[] encodeIteratorSettings(IteratorConfig iterators) {
-TSerializer tser = new TSerializer(new TBinaryProtocol.Factory());
-
try {
+ TSerializer tser = new TSerializer(new TBinaryProtocol.Factory());
return tser.serialize(iterators);
} catch (TException e) {
throw new RuntimeException(e);
@@ -86,9 +85,9 @@ public class SystemIteratorUtil {
}
public static List decodeIteratorSettings(byte[] enc) {
-TDeserializer tdser = new TDeserializer(new TBinaryProtocol.Factory());
IteratorConfig ic = new IteratorConfig();
try {
+ TDeserializer tdser = new TDeserializer(new TBinaryProtocol.Factory());
tdser.deserialize(ic, enc);
} catch (TException e) {
throw new RuntimeException(e);
diff --git
a/core/src/main/java/org/apache/accumulo/core/rpc/FilterTransport.java
b/core/src/main/java/org/apache/accumulo/core/rpc/FilterTransport.java
index f80dc94..3aaba4a 100644
--- a/core/src/main/java/org/apache/accumulo/core/rpc/FilterTransport.java
+++ b/core/src/main/java/org/apache/accumulo/core/rpc/FilterTransport.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.core.rpc;
import static java.util.Objects.requireNonNull;
+import org.apache.thrift.TConfiguration;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
@@ -103,4 +104,19 @@ pub
