(accumulo) branch 2.1 updated: Add scan server prefix to valid zooProp list (#4487)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 0958a5a456 Add scan server prefix to valid zooProp list (#4487)
0958a5a456 is described below
commit 0958a5a4565900a487b7a5ee260ae6099ab8548e
Author: Daniel Roberts
AuthorDate: Tue Apr 23 13:23:58 2024 -0400
Add scan server prefix to valid zooProp list (#4487)
* Add scan server prefix to valid zooProp list
Adds the scan server properties prefix to the list of valid zooProps.
This allows the scan server properties to be modified via the shell.
* Updates fixed properties
Updates the list of fixed properties to include properties that are used
when setting up a server and are never checked again at runtime.
* moved sserver prop
---
.../org/apache/accumulo/core/conf/Property.java| 24 +++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 1db7d9326d..e6b7707235 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -1832,11 +1832,28 @@ public enum Property {
public static final EnumSet fixedProperties = EnumSet.of(
// port options
- GC_PORT, MANAGER_CLIENTPORT, TSERV_CLIENTPORT,
+ GC_PORT, MANAGER_CLIENTPORT, TSERV_CLIENTPORT, SSERV_CLIENTPORT,
SSERV_PORTSEARCH,
+ COMPACTOR_PORTSEARCH, TSERV_PORTSEARCH,
- // tserver cache options
+ // max message options
+ SSERV_MAX_MESSAGE_SIZE, TSERV_MAX_MESSAGE_SIZE,
COMPACTOR_MAX_MESSAGE_SIZE,
+ COMPACTION_COORDINATOR_MAX_MESSAGE_SIZE,
+
+ // block cache options
TSERV_CACHE_MANAGER_IMPL, TSERV_DATACACHE_SIZE, TSERV_INDEXCACHE_SIZE,
- TSERV_SUMMARYCACHE_SIZE,
+ TSERV_SUMMARYCACHE_SIZE, SSERV_DATACACHE_SIZE, SSERV_INDEXCACHE_SIZE,
SSERV_SUMMARYCACHE_SIZE,
+
+ // blocksize options
+ TSERV_DEFAULT_BLOCKSIZE, SSERV_DEFAULT_BLOCKSIZE,
+
+ // sserver specific options
+ SSERVER_SCAN_REFERENCE_EXPIRATION_TIME,
SSERV_CACHED_TABLET_METADATA_EXPIRATION,
+
+ // thread options
+ TSERV_MINTHREADS, TSERV_MINTHREADS_TIMEOUT, SSERV_MINTHREADS,
SSERV_MINTHREADS_TIMEOUT,
+ COMPACTION_COORDINATOR_MINTHREADS,
COMPACTION_COORDINATOR_MINTHREADS_TIMEOUT,
+ MANAGER_MINTHREADS, MANAGER_MINTHREADS_TIMEOUT, COMPACTOR_MINTHREADS,
+ COMPACTOR_MINTHREADS_TIMEOUT,
// others
TSERV_NATIVEMAP_ENABLED, TSERV_SCAN_MAX_OPENFILES,
MANAGER_RECOVERY_WAL_EXISTENCE_CACHE_TIME);
@@ -1862,6 +1879,7 @@ public enum Property {
// white list prefixes
return key.startsWith(Property.TABLE_PREFIX.getKey())
|| key.startsWith(Property.TSERV_PREFIX.getKey())
+|| key.startsWith(Property.SSERV_PREFIX.getKey())
|| key.startsWith(Property.MANAGER_PREFIX.getKey())
|| key.startsWith(Property.MASTER_PREFIX.getKey())
|| key.startsWith(Property.GC_PREFIX.getKey())
(accumulo) branch 2.1 updated: Add visibility to grep searches (#4468)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 3c5326c89f Add visibility to grep searches (#4468)
3c5326c89f is described below
commit 3c5326c89f12741f3b780e13d507d0894e0d2748
Author: alerman
AuthorDate: Mon Apr 22 16:13:14 2024 +
Add visibility to grep searches (#4468)
* Add visibility to grep
* Add unit tests for grep vis
* Update GrepCommand description
---
.../accumulo/core/iterators/user/GrepIterator.java | 2 +-
.../accumulo/core/iterators/user/GrepIteratorTest.java | 18 ++
.../apache/accumulo/shell/commands/GrepCommand.java| 2 +-
.../java/org/apache/accumulo/test/shell/ShellIT.java | 1 +
4 files changed, 21 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
index ff55b2c3da..9e3fa110c8 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
@@ -44,7 +44,7 @@ public class GrepIterator extends Filter {
@Override
public boolean accept(Key k, Value v) {
return match(v.get()) || match(k.getRowData()) ||
match(k.getColumnFamilyData())
-|| match(k.getColumnQualifierData());
+|| match(k.getColumnQualifierData()) ||
match(k.getColumnVisibilityData());
}
protected boolean match(ByteSequence bs) {
diff --git
a/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java
b/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java
index 4a793e4e23..e1c0e30fbd 100644
---
a/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java
@@ -48,9 +48,13 @@ public class GrepIteratorTest {
public void init() {
input = new TreeMap<>();
output = new TreeMap<>();
+
input.put(new Key("abcdef", "xyz", "xyz", 0), new Value("xyz"));
output.put(new Key("abcdef", "xyz", "xyz", 0), new Value("xyz"));
+input.put(new Key("abcdef", "cv", "cv", "colvis", 0), new Value("cv"));
+output.put(new Key("abcdef", "cv", "cv", "colvis", 0), new Value("cv"));
+
input.put(new Key("bdf", "ace", "xyz", 0), new Value("xyz"));
input.put(new Key("bdf", "abcdef", "xyz", 0), new Value("xyz"));
output.put(new Key("bdf", "abcdef", "xyz", 0), new Value("xyz"));
@@ -97,6 +101,20 @@ public class GrepIteratorTest {
GrepIterator.setTerm(is, "def");
gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);
+
+checkEntries(gi, output);
+
+input = new TreeMap<>();
+output = new TreeMap<>();
+
+input.put(new Key("abcdef", "cv", "cv", "colvis", 0), new Value("cv"));
+input.put(new Key("abcdef", "cv", "cv", "nomatch", 0), new Value("cv"));
+output.put(new Key("abcdef", "cv", "cv", "colvis", 0), new Value("cv"));
+
+GrepIterator.setTerm(is, "colvis");
+gi.init(new SortedMapIterator(input), is.getOptions(), null);
+gi.seek(new Range(), EMPTY_COL_FAMS, false);
+
checkEntries(gi, output);
}
}
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
index f4b715fdd8..bc6e10651e 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
@@ -124,7 +124,7 @@ public class GrepCommand extends ScanCommand {
@Override
public String description() {
-return "searches each row, column family, column qualifier and value in a"
+return "searches each row, column family, column qualifier, visibility,
and value in a"
+ " table for a substring (not a regular expression), in parallel, on
the server side";
}
diff --git a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
index 4f592791f7..650ebac7c3 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
@@ -464,6 +464,7 @@ public class ShellIT extends SharedMiniClusterBase {
String expected = "r f:q [vis]\tv";
String expectedTimestamp = "r f:q [vis] 0\tv";
exec("grep", false, "No terms specified");
+exec("grep vis", true, expected);
exec("grep non_matching_string", true, "");
// historically, showing few did not pertain to ColVis or Timestamp
exec("grep r", true, expected);
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 221735d0456a9245f5c801d749c0ffe4706aa68b Merge: 996952ff55 c4ceb7e7a9 Author: Daniel Roberts AuthorDate: Mon Apr 1 15:45:33 2024 + Merge branch 'main' into elasticity .../java/org/apache/accumulo/core/util/Retry.java | 53 .../org/apache/accumulo/core/util/RetryTest.java | 98 ++ pom.xml| 2 +- .../shell/commands/HistoryCommandTest.java | 11 +-- 4 files changed, 158 insertions(+), 6 deletions(-)
(accumulo) branch elasticity updated (996952ff55 -> 221735d045)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 996952ff55 Added missing Fate.shutdown in FateOpsCommandsIT (#4428) add 5d741657e7 Add duration based maxRetries option to Retry class (#4337) add 01ccb96472 Fix Illegal Reflective Access Warning in Shell (#4427) add c4ceb7e7a9 Merge branch '2.1' new 221735d045 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/util/Retry.java | 53 .../org/apache/accumulo/core/util/RetryTest.java | 98 ++ pom.xml| 2 +- .../shell/commands/HistoryCommandTest.java | 11 +-- 4 files changed, 158 insertions(+), 6 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c4ceb7e7a9f183059d753386d6a09cd880f1bb08 Merge: 5d741657e7 01ccb96472 Author: Daniel Roberts AuthorDate: Mon Apr 1 15:25:19 2024 + Merge branch '2.1' pom.xml | 2 +- .../apache/accumulo/shell/commands/HistoryCommandTest.java| 11 ++- 2 files changed, 7 insertions(+), 6 deletions(-)
(accumulo) branch main updated (5d741657e7 -> c4ceb7e7a9)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5d741657e7 Add duration based maxRetries option to Retry class (#4337) add 01ccb96472 Fix Illegal Reflective Access Warning in Shell (#4427) new c4ceb7e7a9 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 2 +- .../apache/accumulo/shell/commands/HistoryCommandTest.java| 11 ++- 2 files changed, 7 insertions(+), 6 deletions(-)
(accumulo) branch 2.1 updated: Fix Illegal Reflective Access Warning in Shell (#4427)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 01ccb96472 Fix Illegal Reflective Access Warning in Shell (#4427)
01ccb96472 is described below
commit 01ccb9647269bbf216043721afa298721dd80699
Author: Daniel Roberts
AuthorDate: Mon Apr 1 10:58:18 2024 -0400
Fix Illegal Reflective Access Warning in Shell (#4427)
Jline 3.24.1 throws an illegal reflective access warning when starting
an accumulo shell using java 11.
Updates jline to use 3.25.1 which includes the fix.
---
pom.xml | 2 +-
.../apache/accumulo/shell/commands/HistoryCommandTest.java| 11 ++-
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/pom.xml b/pom.xml
index fb747c50a8..fbffd60171 100644
--- a/pom.xml
+++ b/pom.xml
@@ -624,7 +624,7 @@
org.jline
jline
-3.24.1
+3.25.1
org.latencyutils
diff --git
a/shell/src/test/java/org/apache/accumulo/shell/commands/HistoryCommandTest.java
b/shell/src/test/java/org/apache/accumulo/shell/commands/HistoryCommandTest.java
index 4228eaa7e2..6e45768f8f 100644
---
a/shell/src/test/java/org/apache/accumulo/shell/commands/HistoryCommandTest.java
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/HistoryCommandTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.shell.commands;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
@@ -38,7 +39,7 @@ import org.jline.reader.LineReaderBuilder;
import org.jline.reader.impl.DefaultExpander;
import org.jline.reader.impl.history.DefaultHistory;
import org.jline.terminal.Terminal;
-import org.jline.terminal.TerminalBuilder;
+import org.jline.terminal.impl.ExternalTerminal;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -69,8 +70,8 @@ public class HistoryCommandTest {
baos = new ByteArrayOutputStream();
String input = String.format("!1%n"); // Construct a platform dependent
new-line
-terminal = TerminalBuilder.builder().system(false)
-.streams(new ByteArrayInputStream(input.getBytes()), baos).build();
+terminal = new ExternalTerminal("shell", "ansi", new
ByteArrayInputStream(input.getBytes()),
+baos, UTF_8);
reader =
LineReaderBuilder.builder().history(history).terminal(terminal).build();
shell = new Shell(reader);
@@ -80,8 +81,8 @@ public class HistoryCommandTest {
public void testCorrectNumbering() throws IOException {
command.execute("", cl, shell);
terminal.writer().flush();
-
-assertTrue(baos.toString().contains("2: bar"));
+assertTrue(baos.toString().contains("2: bar"),
+"History order is not correct: " + baos.toString());
}
@Test
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 171a1e144caa9d2eb382846d2dc45a424487d3ec
Merge: 162b8effb1 92331ea113
Author: Daniel Roberts
AuthorDate: Tue Mar 12 15:17:40 2024 +
Merge branch '2.1'
.../accumulo/core/file/rfile/GenerateSplits.java | 32 +--
.../core/file/rfile/GenerateSplitsTest.java| 64 +++---
2 files changed, 82 insertions(+), 14 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
index 5928ade21d,865210a970..7ee151f0a2
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
@@@ -137,19 -143,33 +144,33 @@@ public class GenerateSplits implements
Path path = new Path(file);
fs = PrintInfo.resolveFS(log, hadoopConf, path);
// get all the files in the directory
- filePaths.addAll(getFiles(fs, path));
+ files.addAll(getFiles(fs, path));
}
-if (filePaths.isEmpty()) {
+if (files.isEmpty()) {
throw new IllegalArgumentException("No files were found in " +
opts.files);
} else {
- log.trace("Found the following files: {}", filePaths);
+ log.trace("Found the following files: {}", files);
}
+ if (!encode) {
+ // Generate the allowed Character set
+ for (int i = 0; i < 10; i++) {
+ // 0-9
+ allowedChars.add((char) (i + 48));
+ }
+ for (int i = 0; i < 26; i++) {
+ // Uppercase A-Z
+ allowedChars.add((char) (i + 65));
+ // Lowercase a-z
+ allowedChars.add((char) (i + 97));
+ }
+ }
+
// if no size specified look at indexed keys first
if (opts.splitSize == 0) {
- splits = getIndexKeys(siteConf, hadoopConf, fs, filePaths,
requestedNumSplits, encode,
- cryptoService);
+ splits =
+ getIndexKeys(siteConf, hadoopConf, fs, files, requestedNumSplits,
encode, cryptoService);
// if there weren't enough splits indexed, try again with size = 0
if (splits.size() < requestedNumSplits) {
log.info("Only found {} indexed keys but need {}. Doing a full scan
on files {}",
(accumulo) branch main updated (162b8effb1 -> 171a1e144c)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 162b8effb1 Replace long + TimeUnit with Duration in ReadOnlyTStore.unreserve() (#4358) add 92331ea113 Throw error when non-standard chars exist (#4348) new 171a1e144c Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/file/rfile/GenerateSplits.java | 32 +-- .../core/file/rfile/GenerateSplitsTest.java| 64 +++--- 2 files changed, 82 insertions(+), 14 deletions(-)
(accumulo) branch 2.1 updated: Throw error when non-standard chars exist (#4348)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 92331ea113 Throw error when non-standard chars exist (#4348)
92331ea113 is described below
commit 92331ea113b12c0a833a0a5ea714294832c6d822
Author: Daniel Roberts
AuthorDate: Tue Mar 12 11:12:53 2024 -0400
Throw error when non-standard chars exist (#4348)
* Throw error when non-standard chars exist
Adds an explicit error message when base64 coding should have been used.
Removes specical handling of `\` character and replaces it with an
allowed character set.
-
Co-authored-by: Keith Turner
---
.../accumulo/core/file/rfile/GenerateSplits.java | 32 +--
.../core/file/rfile/GenerateSplitsTest.java| 64 +++---
2 files changed, 82 insertions(+), 14 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
index 1812358a59..865210a970 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -72,6 +73,10 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
public class GenerateSplits implements KeywordExecutable {
private static final Logger log =
LoggerFactory.getLogger(GenerateSplits.class);
+ private static final Set allowedChars = new HashSet<>();
+
+ private static final String encodeFlag = "-b64";
+
static class Opts extends ConfigOpts {
@Parameter(names = {"-n", "--num"},
description = "The number of split points to generate. Can be used to
create n+1 tablets. Cannot use with the split size option.")
@@ -81,7 +86,8 @@ public class GenerateSplits implements KeywordExecutable {
description = "The minimum split size in uncompressed bytes. Cannot
use with num splits option.")
public long splitSize = 0;
-@Parameter(names = {"-b64", "--base64encoded"}, description = "Base 64
encode the split points")
+@Parameter(names = {encodeFlag, "--base64encoded"},
+description = "Base 64 encode the split points")
public boolean base64encode = false;
@Parameter(names = {"-sf", "--splits-file"}, description = "Output the
splits to a file")
@@ -89,6 +95,7 @@ public class GenerateSplits implements KeywordExecutable {
@Parameter(description = "[ ...] -n
| -ss ")
public List files = new ArrayList<>();
+
}
@Override
@@ -145,6 +152,20 @@ public class GenerateSplits implements KeywordExecutable {
log.trace("Found the following files: {}", filePaths);
}
+if (!encode) {
+ // Generate the allowed Character set
+ for (int i = 0; i < 10; i++) {
+// 0-9
+allowedChars.add((char) (i + 48));
+ }
+ for (int i = 0; i < 26; i++) {
+// Uppercase A-Z
+allowedChars.add((char) (i + 65));
+// Lowercase a-z
+allowedChars.add((char) (i + 97));
+ }
+}
+
// if no size specified look at indexed keys first
if (opts.splitSize == 0) {
splits = getIndexKeys(siteConf, hadoopConf, fs, filePaths,
requestedNumSplits, encode,
@@ -256,16 +277,15 @@ public class GenerateSplits implements KeywordExecutable {
if (encode) {
return Base64.getEncoder().encodeToString(bytes);
} else {
- // drop non printable characters
StringBuilder sb = new StringBuilder();
for (byte aByte : bytes) {
int c = 0xff & aByte;
-if (c == '\\') {
- sb.append("");
-} else if (c >= 32 && c <= 126) {
+if (allowedChars.contains((char) c)) {
sb.append((char) c);
} else {
- log.debug("Dropping non printable char: \\x{}",
Integer.toHexString(c));
+ // Fail if non-printable characters are detected.
+ throw new UnsupportedOperationException("Non printable char: \\x" +
Integer.toHexString(c)
+ + " detected. Must use Base64 encoded output. The behavior
around non printable chars changed in 2.1.3 to throw an error, the previous
behavior was likely to cause bugs.");
}
}
return sb.toString();
diff --git
a/core/src/test/java/org/apache/accumulo/core/file/rfile/GenerateSplitsTest.java
b/core/src/test/java/o
(accumulo-examples) branch helm-chart-collab updated: Added files for 3.1 accumulo
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch helm-chart-collab in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/helm-chart-collab by this push: new 7e85355 Added files for 3.1 accumulo 7e85355 is described below commit 7e85355a8aff6b5e78dcd6c74b80fd99b16838ed Author: Daniel Roberts AuthorDate: Thu Feb 29 18:58:12 2024 + Added files for 3.1 accumulo --- helm/k8s-files/accumulo-compactor.yaml | 62 helm/k8s-files/accumulo-config.yaml | 121 +++ helm/k8s-files/accumulo-coordinator.yaml | 64 helm/k8s-files/accumulo-gc.yaml | 63 helm/k8s-files/accumulo-init.yaml| 61 helm/k8s-files/accumulo-manager.yaml | 63 helm/k8s-files/accumulo-monitor.yaml | 62 helm/k8s-files/accumulo-ns.yaml | 7 ++ helm/k8s-files/accumulo-secrets.yaml | 10 +++ helm/k8s-files/accumulo-sserver.yaml | 63 helm/k8s-files/accumulo-tserver.yaml | 63 helm/k8s-files/minio-dev.yaml| 44 +++ helm/k8s-files/minio-svc.yaml| 18 + 13 files changed, 701 insertions(+) diff --git a/helm/k8s-files/accumulo-compactor.yaml b/helm/k8s-files/accumulo-compactor.yaml new file mode 100644 index 000..e2ca412 --- /dev/null +++ b/helm/k8s-files/accumulo-compactor.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-compactor + namespace: accumulo + labels: +app: accumulo-compactor +spec: + replicas: 1 + selector: +matchLabels: + app: accumulo-compactor + template: +metadata: + labels: +app: accumulo-compactor +spec: + containers: + - name: accumulo-compactor +image: accumulo-s3-fs:2.1.0 +imagePullPolicy: Never +command: ["/bin/bash", "-c"] +args: ["accumulo compactor -o compactor.queue=DCQ1 -o general.process.bind.addr=$(hostname -i)" ] +ports: +- containerPort: 9133 +resources: + requests: +cpu: 100m +memory: "2048Mi" + limits: +cpu: 200m +memory: "4096Mi" +env: +- name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" +volumeMounts: +- name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" +- name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" +- name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" +- name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" +configMap: + name: "accumulo-client-properties" + - name: "config" +configMap: + name: "accumulo-properties" + - name: "logging" +configMap: + name: "accumulo-logging" + - name: "core-site" +configMap: + name: "core-site" diff --git a/helm/k8s-files/accumulo-config.yaml b/helm/k8s-files/accumulo-config.yaml new file mode 100644 index 000..0d5a02b --- /dev/null +++ b/helm/k8s-files/accumulo-config.yaml @@ -0,0 +1,121 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: accumulo-client-properties + namespace: accumulo +data: + accumulo-client.properties: | +auth.type=password +auth.principal=root +instance.name=s3test +instance.zookeepers=bitnami-zookeeper.zookeeper.svc.cluster.local:2181 +rpc.transport.idle.timeout=60s +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: accumulo-properties + namespace: accumulo +data: + accumulo.properties: | +general.rpc.timeout=240s +instance.secret=s3test +instance.zookeeper.host=bitnami-zookeeper.zookeeper.svc.cluster.local:2181 + manager.wal.closer.implementation=org.apache.accumulo.server.manager.recovery.NoOpLogCloser +table.durability=sync +tserver.memory.maps.native.enabled=false +tserver.walog.max.size=512M +tserver.port.search=true +## For S3 you must define volumes for accumulo and its write ahead logs. Replace the defaule instance volumes at the top +## with the example below. Make sure the
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3f3da5a5da9cf500777d4c4b0d5a1c6556557dd8
Merge: 5bec92248c 9817474333
Author: Daniel Roberts
AuthorDate: Fri Mar 1 01:21:57 2024 +
Merge branch 'main' into elasticity
.../core/clientImpl/InstanceOperationsImpl.java| 10 +-
.../core/clientImpl/NamespaceOperationsImpl.java | 9 +-
.../core/clientImpl/TableOperationsImpl.java | 14 +-
.../TabletServerBatchReaderIterator.java | 8 +-
.../core/clientImpl/TabletServerBatchWriter.java | 9 +-
.../accumulo/core/clientImpl/ThriftScanner.java| 8 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 9 +-
.../accumulo/core/fate/zookeeper/ZooReader.java| 8 +-
.../spi/compaction/SimpleCompactionDispatcher.java | 5 +-
.../java/org/apache/accumulo/core/util/Retry.java | 112 +++---
.../org/apache/accumulo/core/util/RetryTest.java | 163 +++--
.../hadoopImpl/mapred/AccumuloRecordReader.java| 10 +-
.../hadoopImpl/mapreduce/AccumuloRecordReader.java | 10 +-
.../server/compaction/RetryableThriftCall.java | 8 +-
.../metadata/ConditionalTabletsMutatorImpl.java| 11 +-
.../java/org/apache/accumulo/manager/Manager.java | 7 +-
.../coordinator/CompactionCoordinator.java | 8 +-
.../coordinator/commit/CommitCompaction.java | 10 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 11 +-
.../manager/tableOps/bulkVer2/TabletRefresher.java | 10 +-
.../manager/tableOps/compact/CompactionDriver.java | 10 +-
.../org/apache/accumulo/tserver/TabletServer.java | 19 +--
.../accumulo/tserver/session/SessionManager.java | 9 +-
23 files changed, 232 insertions(+), 246 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 6e558b9e3b,8435df2a40..bd708d2b19
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@@ -23,19 -23,10 +23,18 @@@ import static com.google.common.util.co
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
- import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toSet;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import static org.apache.accumulo.core.util.Validators.EXISTING_TABLE_NAME;
import static org.apache.accumulo.core.util.Validators.NEW_TABLE_NAME;
@@@ -1876,74 -1882,26 +1876,74 @@@ public class TableOperationsImpl extend
rangeList = new ArrayList<>(ranges);
}
-Map>> binnedRanges = new HashMap<>();
-
+ClientTabletCache locator = ClientTabletCache.getInstance(context,
tableId);
locator.invalidateCache();
- Retry retry = Retry.builder().infiniteRetries().retryAfter(100,
MILLISECONDS)
- .incrementBy(100, MILLISECONDS).maxWait(2, SECONDS).backOffFactor(1.5)
- .logInterval(3, MINUTES).createRetry();
+ Retry retry =
Retry.builder().infiniteRetries().retryAfter(Duration.ofMillis(100))
+
.incrementBy(Duration.ofMillis(100)).maxWait(Duration.ofSeconds(2)).backOffFactor(1.5)
+ .logInterval(Duration.ofMinutes(3)).createRetry();
-while (!locator.binRanges(context, rangeList, binnedRanges).isEmpty()) {
- context.requireTableExists(tableId, tableName);
- context.requireNotOffline(tableId, tableName);
- binnedRanges.clear();
- try {
-retry.waitForNextAttempt(log,
-String.format("locating tablets in table %s(%s) for %d ranges",
tableName, tableId,
-rangeList.size()));
- } catch (InterruptedException e) {
-throw new IllegalStateExcepti
(accumulo) branch elasticity updated (5bec92248c -> 3f3da5a5da)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5bec92248c update IdleProcessMetricsIT to work w/ recent changes (#4326) add aed3919e46 Use Duration in Retry instead of long+TimeUnit (#4301) add 48f150045b Fixes compaction dispatcher documentation (#4314) add 9817474333 Merge branch '2.1' new 3f3da5a5da Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/clientImpl/InstanceOperationsImpl.java| 10 +- .../core/clientImpl/NamespaceOperationsImpl.java | 9 +- .../core/clientImpl/TableOperationsImpl.java | 14 +- .../TabletServerBatchReaderIterator.java | 8 +- .../core/clientImpl/TabletServerBatchWriter.java | 9 +- .../accumulo/core/clientImpl/ThriftScanner.java| 8 +- .../accumulo/core/clientImpl/bulk/BulkImport.java | 9 +- .../accumulo/core/fate/zookeeper/ZooReader.java| 8 +- .../spi/compaction/SimpleCompactionDispatcher.java | 5 +- .../java/org/apache/accumulo/core/util/Retry.java | 112 +++--- .../org/apache/accumulo/core/util/RetryTest.java | 163 +++-- .../hadoopImpl/mapred/AccumuloRecordReader.java| 10 +- .../hadoopImpl/mapreduce/AccumuloRecordReader.java | 10 +- .../server/compaction/RetryableThriftCall.java | 8 +- .../metadata/ConditionalTabletsMutatorImpl.java| 11 +- .../java/org/apache/accumulo/manager/Manager.java | 7 +- .../coordinator/CompactionCoordinator.java | 8 +- .../coordinator/commit/CommitCompaction.java | 10 +- .../tableOps/bulkVer2/CleanUpBulkImport.java | 11 +- .../manager/tableOps/bulkVer2/TabletRefresher.java | 10 +- .../manager/tableOps/compact/CompactionDriver.java | 10 +- .../org/apache/accumulo/tserver/TabletServer.java | 19 +-- .../accumulo/tserver/session/SessionManager.java | 9 +- 23 files changed, 232 insertions(+), 246 deletions(-)
(accumulo) branch main updated (aed3919e46 -> 9817474333)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from aed3919e46 Use Duration in Retry instead of long+TimeUnit (#4301) add 48f150045b Fixes compaction dispatcher documentation (#4314) new 9817474333 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/core/Constants.java | 1 + .../java/org/apache/accumulo/core/conf/Property.java | 18 -- .../spi/compaction/SimpleCompactionDispatcher.java | 10 +++--- .../accumulo/server/conf/CheckCompactionConfig.java| 4 +++- .../test/compaction/BadCompactionServiceConfigIT.java | 4 +++- 5 files changed, 26 insertions(+), 11 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 981747433306236511fb652c4b7ebe5a461046ae
Merge: aed3919e46 48f150045b
Author: Daniel Roberts
AuthorDate: Fri Mar 1 00:51:19 2024 +
Merge branch '2.1'
.../main/java/org/apache/accumulo/core/Constants.java | 1 +
.../java/org/apache/accumulo/core/conf/Property.java | 18 --
.../spi/compaction/SimpleCompactionDispatcher.java | 10 +++---
.../accumulo/server/conf/CheckCompactionConfig.java| 4 +++-
.../test/compaction/BadCompactionServiceConfigIT.java | 4 +++-
5 files changed, 26 insertions(+), 11 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6cb1207f40,1db7d9326d..e2afb7defb
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -637,25 -685,23 +639,29 @@@ public enum Property
PropertyType.JSON,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
+ @Deprecated(since = "3.1")
-
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
+ TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER(
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME +
".planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Planner for default compaction service.", "2.1.0"),
+ @Deprecated(since = "3.1")
-
TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT("tserver.compaction.major.service.default.rate.limit",
- "0B", PropertyType.BYTES,
+ TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT(
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME +
".rate.limit", "0B",
+ PropertyType.BYTES,
"Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
+ + " compactions in this compaction service, or 0B for unlimited.
This property has"
+ + " been deprecated in anticipation of it being removed in a future
release that"
+ + " removes the rate limiting feature.",
"2.1.0"),
+ @Deprecated(since = "3.1")
TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN(
- "tserver.compaction.major.service.default.planner.opts.maxOpen", "10",
PropertyType.COUNT,
- "The maximum number of files a compaction will open.", "2.1.0"),
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME
+ + ".planner.opts.maxOpen",
+ "10", PropertyType.COUNT, "The maximum number of files a compaction
will open.", "2.1.0"),
+ @Deprecated(since = "3.1")
TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS(
- "tserver.compaction.major.service.default.planner.opts.executors",
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME
+ + ".planner.opts.executors",
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':2},{'name':'medium','type':'internal','maxSize':'128M','numThreads':2},{'name':'large','type':'internal','numThreads':2}]"
.replaceAll("'", "\""),
PropertyType.STRING,
(accumulo) branch 2.1 updated: Fixes compaction dispatcher documentation (#4314)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 48f150045b Fixes compaction dispatcher documentation (#4314)
48f150045b is described below
commit 48f150045b770e1ba08005ab831ed772a5e507ad
Author: Daniel Roberts
AuthorDate: Thu Feb 29 19:45:35 2024 -0500
Fixes compaction dispatcher documentation (#4314)
Fixes the documentation which directed users to use "selected" while the
code actually checked for "selector".
Also replaced the hardcoded "default" compaction service name string
with a constant.
-
Co-authored-by: Keith Turner
---
.../main/java/org/apache/accumulo/core/Constants.java | 1 +
.../java/org/apache/accumulo/core/conf/Property.java | 18 --
.../spi/compaction/SimpleCompactionDispatcher.java | 10 +++---
.../accumulo/server/conf/CheckCompactionConfig.java| 4 +++-
.../test/compaction/BadCompactionServiceConfigIT.java | 4 +++-
5 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java
b/core/src/main/java/org/apache/accumulo/core/Constants.java
index 00b1a2fd18..c31e205585 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -129,4 +129,5 @@ public class Constants {
public static final String HDFS_TABLES_DIR = "/tables";
public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
+ public static final String DEFAULT_COMPACTION_SERVICE_NAME = "default";
}
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 08a93ae6b3..1db7d9326d 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.core.conf;
+import static
org.apache.accumulo.core.Constants.DEFAULT_COMPACTION_SERVICE_NAME;
+
import java.lang.annotation.Annotation;
import java.util.Arrays;
import java.util.EnumSet;
@@ -683,19 +685,23 @@ public enum Property {
PropertyType.JSON,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
-
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
+ TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER(
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME +
".planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Planner for default compaction service.", "2.1.0"),
-
TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT("tserver.compaction.major.service.default.rate.limit",
- "0B", PropertyType.BYTES,
+ TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT(
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME +
".rate.limit", "0B",
+ PropertyType.BYTES,
"Maximum number of bytes to read or write per second over all major"
+ " compactions in this compaction service, or 0B for unlimited.",
"2.1.0"),
TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN(
- "tserver.compaction.major.service.default.planner.opts.maxOpen", "10",
PropertyType.COUNT,
- "The maximum number of files a compaction will open.", "2.1.0"),
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME
+ + ".planner.opts.maxOpen",
+ "10", PropertyType.COUNT, "The maximum number of files a compaction will
open.", "2.1.0"),
TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS(
- "tserver.compaction.major.service.default.planner.opts.executors",
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME
+ + ".planner.opts.executors",
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':2},{'name':'medium','type':'internal','maxSize':'128M','numThreads':2},{'name':'large','type':'internal','numThreads':2}]"
.replaceAll("'", "\""),
PropertyType.STRING,
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/SimpleCompactionDispatcher.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/SimpleCompactionDispatcher.java
index 93aff66547..44d6f51247 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/SimpleCompactionDispatcher.java
+++
b/
(accumulo) branch elasticity updated: Reduce compaction properties (#4311)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new cf12b6a84a Reduce compaction properties (#4311)
cf12b6a84a is described below
commit cf12b6a84ad7cee506332d62af33e438f0db43ac
Author: Daniel Roberts
AuthorDate: Wed Feb 28 11:08:33 2024 -0500
Reduce compaction properties (#4311)
Removes the "meta" and "root" compaction services and replaces them with
a single "default" compaction service.
Adds a constant for referencing the "default" compaction service name.
Renamed the DefaultCompactionPlanner to RatioBasedCompactionPlanner.
---
.../java/org/apache/accumulo/core/Constants.java | 1 +
.../core/client/admin/InstanceOperations.java | 2 +-
.../org/apache/accumulo/core/conf/Property.java| 47 --
...anner.java => RatioBasedCompactionPlanner.java} | 4 +-
.../spi/compaction/SimpleCompactionDispatcher.java | 5 ++-
.../accumulo/core/spi/compaction/package-info.java | 2 +-
...t.java => RatioBasedCompactionPlannerTest.java} | 21 +-
.../compaction/CompactionServicesConfigTest.java | 20 -
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 10 -
.../server/conf/CheckCompactionConfig.java | 12 +++---
.../accumulo/server/init/InitialConfiguration.java | 10 +
.../server/conf/CheckCompactionConfigTest.java | 20 -
.../compaction/BadCompactionServiceConfigIT.java | 14 ---
.../test/compaction/CompactionConfigChangeIT.java | 4 +-
.../CompactionPriorityQueueMetricsIT.java | 4 +-
.../compaction/ExternalCompactionTestUtils.java| 18 -
.../accumulo/test/functional/CompactionIT.java | 8 ++--
.../apache/accumulo/test/shell/ConfigSetIT.java| 5 ++-
18 files changed, 92 insertions(+), 115 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java
b/core/src/main/java/org/apache/accumulo/core/Constants.java
index a35cf61707..a06675f846 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -124,4 +124,5 @@ public class Constants {
public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
public static final String DEFAULT_RESOURCE_GROUP_NAME = "default";
+ public static final String DEFAULT_COMPACTION_SERVICE_NAME = "default";
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index 4cb3f6d196..cc85ecc524 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@ -90,7 +90,7 @@ public interface InstanceOperations {
* var planner =
currProps.get("compaction.service.default.planner");
* //This code will only change the compaction planner if its
currently set to default settings.
* //The endsWith() function was used to make the example
short, would be better to use equals().
- * if(planner != null &&
planner.endsWith("DefaultCompactionPlanner") {
+ * if(planner != null &&
planner.endsWith("RatioBasedCompactionPlanner") {
* // tservers will eventually see these compaction planner
changes and when they do they will see all of the changes at once
* currProps.keySet().removeIf(
*prop ->
prop.startsWith("compaction.service.default.planner.opts."));
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index f48253a812..a3057f9d36 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.core.conf;
+import static
org.apache.accumulo.core.Constants.DEFAULT_COMPACTION_SERVICE_NAME;
+
import java.lang.annotation.Annotation;
import java.util.Arrays;
import java.util.EnumSet;
@@ -32,7 +34,7 @@ import org.apache.accumulo.core.file.rfile.RFile;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator;
import org.apache.accumulo.core.metadata.AccumuloTable;
-import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner;
+import org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner;
import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
import org.apache.accumu
(accumulo-access) 01/01: Merge pull request #62 from ddanielr/Remove-contrib-ref
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit 2da31d3643916c020b14c9e61fa632543dbc56b7 Merge: e0abc5d 374a81e Author: Daniel Roberts AuthorDate: Sat Feb 17 13:00:15 2024 -0500 Merge pull request #62 from ddanielr/Remove-contrib-ref Update README to remove `contrib` project ref src/it/antlr4-example/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo-access) branch main updated (e0abc5d -> 2da31d3)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git from e0abc5d Merge branch '1.0.0-beta-rc4-next' add 374a81e Update README to remove `contrib` project ref new 2da31d3 Merge pull request #62 from ddanielr/Remove-contrib-ref The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/it/antlr4-example/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch elasticity updated: Fix build from broken merge
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new abad2b2be9 Fix build from broken merge
abad2b2be9 is described below
commit abad2b2be9306de653924d88cb5ce77011b13741
Author: Daniel Roberts
AuthorDate: Fri Feb 16 23:20:34 2024 +
Fix build from broken merge
---
.../accumulo/manager/compaction/coordinator/CompactionCoordinator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index bf2e9009f3..3bf393d1eb 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -296,7 +296,7 @@ public class CompactionCoordinator
}
protected long getMissingCompactorWarningTime() {
-return
getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME) * 3;
+return
this.ctx.getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME)
* 3;
}
protected long getTServerCheckInterval() {
(accumulo) branch elasticity updated (3de72f63c4 -> 9f073b4838)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 3de72f63c4 fixes FateStarvationIT (#4278) add 3298d6db89 Add Compaction Job Min & Max Wait properties (#4223) add 9f430a2695 Merge branch '2.1' add 4886e821ed Remove unused var added by merge new 9f073b4838 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/core/conf/Property.java | 4 ++-- .../main/java/org/apache/accumulo/compactor/Compactor.java | 14 -- .../compaction/coordinator/CompactionCoordinator.java | 3 +-- 3 files changed, 11 insertions(+), 10 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9f073b4838ee87847d51fc94878e6aeffae85a78
Merge: 3de72f63c4 4886e821ed
Author: Daniel Roberts
AuthorDate: Fri Feb 16 22:39:39 2024 +
Merge branch 'main' into elasticity
.../main/java/org/apache/accumulo/core/conf/Property.java | 4 ++--
.../main/java/org/apache/accumulo/compactor/Compactor.java | 14 --
.../compaction/coordinator/CompactionCoordinator.java | 3 +--
3 files changed, 11 insertions(+), 10 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 22c2606c85,6cb1207f40..394b79746e
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1112,12 -1130,20 +1112,12 @@@ public enum Property
COMPACTOR_MIN_JOB_WAIT_TIME("compactor.wait.time.job.min", "1s",
PropertyType.TIMEDURATION,
"The minimum amount of time to wait between checks for the next
compaction job, backing off"
+ "exponentially until COMPACTOR_MAX_JOB_WAIT_TIME is reached.",
- "4.0.0"),
+ "2.1.3"),
- @Experimental
COMPACTOR_MAX_JOB_WAIT_TIME("compactor.wait.time.job.max", "5m",
PropertyType.TIMEDURATION,
"Compactors do exponential backoff when their request for work
repeatedly come back empty. "
+ "This is the maximum amount of time to wait between checks for
the next compaction job.",
- "4.0.0"),
+ "2.1.3"),
@Experimental
- COMPACTOR_PORTSEARCH("compactor.port.search", "false", PropertyType.BOOLEAN,
- "If the compactor.port.client is in use, search higher ports until one
is available.",
- "2.1.0"),
- @Experimental
- COMPACTOR_CLIENTPORT("compactor.port.client", "9133", PropertyType.PORT,
- "The port used for handling client connections on the compactor
servers.", "2.1.0"),
- @Experimental
COMPACTOR_MINTHREADS("compactor.threads.minimum", "1", PropertyType.COUNT,
"The minimum number of threads to use to handle incoming requests.",
"2.1.0"),
@Experimental
diff --cc
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 1a1c2ece58,b81c30dee3..114dd59519
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -589,15 -582,16 +589,17 @@@ public class Compactor extends Abstract
}
protected long getWaitTimeBetweenCompactionChecks() {
-// get the total number of compactors assigned to this queue
-int numCompactors = ExternalCompactionUtil.countCompactors(queueName,
getContext());
+// get the total number of compactors assigned to this group
+int numCompactors =
+ExternalCompactionUtil.countCompactors(this.getResourceGroup(),
getContext());
- // Aim for around 3 compactors checking in every second
- long sleepTime = numCompactors * 1000L / 3;
- // Ensure a compactor sleeps at least around a second
- sleepTime = Math.max(1000, sleepTime);
- // Ensure a compactor sleep not too much more than 5 mins
- sleepTime = Math.min(300_000L, sleepTime);
+ long minWait =
getConfiguration().getTimeInMillis(Property.COMPACTOR_MIN_JOB_WAIT_TIME);
+ // Aim for around 3 compactors checking in per min wait time.
+ long sleepTime = numCompactors * minWait / 3;
+ // Ensure a compactor waits at least the minimum time
+ sleepTime = Math.max(minWait, sleepTime);
+ // Ensure a sleeping compactor has a configurable max sleep time
+ sleepTime =
Math.min(getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME),
+ sleepTime);
// Add some random jitter to the sleep time, that averages out to sleep
time. This will spread
// compactors out evenly over time.
sleepTime = (long) (.9 * sleepTime + sleepTime * .2 *
RANDOM.get().nextDouble());
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index fdbf009352,00..bf2e9009f3
mode 100644,00..100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@@ -1,1012 -1,0 +1,1011 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The
(accumulo) branch main updated: Remove unused var added by merge
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4886e821ed Remove unused var added by merge
4886e821ed is described below
commit 4886e821ed87ab1934e03540826c6b20863101d3
Author: Daniel Roberts
AuthorDate: Fri Feb 16 22:27:55 2024 +
Remove unused var added by merge
---
.../main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 97b09b5e89..1af899acd5 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -104,7 +104,6 @@ public class CompactionCoordinator extends AbstractServer
implements CompactionCoordinatorService.Iface, LiveTServerSet.Listener {
private static final Logger LOG =
LoggerFactory.getLogger(CompactionCoordinator.class);
- private static final long TIME_BETWEEN_GC_CHECKS = 5000;
protected static final QueueSummaries QUEUE_SUMMARIES = new QueueSummaries();
/*
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9f430a26956449133d13c1a35c367df7ce33dc7e
Merge: 978f8b9eca 3298d6db89
Author: Daniel Roberts
AuthorDate: Fri Feb 16 22:12:55 2024 +
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java | 10 ++
.../accumulo/coordinator/CompactionCoordinator.java | 5 ++---
.../org/apache/accumulo/compactor/Compactor.java| 21 ++---
3 files changed, 26 insertions(+), 10 deletions(-)
diff --cc
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 00956edbf1,1434d2a1f1..b81c30dee3
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -579,15 -592,17 +584,17 @@@ public class Compactor extends Abstract
protected long getWaitTimeBetweenCompactionChecks() {
// get the total number of compactors assigned to this queue
int numCompactors = ExternalCompactionUtil.countCompactors(queueName,
getContext());
- // Aim for around 3 compactors checking in every second
- long sleepTime = numCompactors * 1000L / 3;
- // Ensure a compactor sleeps at least around a second
- sleepTime = Math.max(1000, sleepTime);
- // Ensure a compactor sleep not too much more than 5 mins
- sleepTime = Math.min(300_000L, sleepTime);
+ long minWait =
getConfiguration().getTimeInMillis(Property.COMPACTOR_MIN_JOB_WAIT_TIME);
+ // Aim for around 3 compactors checking in per min wait time.
+ long sleepTime = numCompactors * minWait / 3;
+ // Ensure a compactor waits at least the minimum time
+ sleepTime = Math.max(minWait, sleepTime);
+ // Ensure a sleeping compactor has a configurable max sleep time
+ sleepTime =
Math.min(getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME),
+ sleepTime);
// Add some random jitter to the sleep time, that averages out to sleep
time. This will spread
// compactors out evenly over time.
-sleepTime = (long) (.9 * sleepTime + sleepTime * .2 *
random.nextDouble());
+sleepTime = (long) (.9 * sleepTime + sleepTime * .2 *
RANDOM.get().nextDouble());
LOG.trace("Sleeping {}ms based on {} compactors", sleepTime,
numCompactors);
return sleepTime;
}
(accumulo) branch main updated (978f8b9eca -> 9f430a2695)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 978f8b9eca Merge branch '2.1' add 3298d6db89 Add Compaction Job Min & Max Wait properties (#4223) new 9f430a2695 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/conf/Property.java | 10 ++ .../accumulo/coordinator/CompactionCoordinator.java | 5 ++--- .../org/apache/accumulo/compactor/Compactor.java| 21 ++--- 3 files changed, 26 insertions(+), 10 deletions(-)
(accumulo) branch 2.1 updated: Add Compaction Job Min & Max Wait properties (#4223)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 3298d6db89 Add Compaction Job Min & Max Wait properties (#4223)
3298d6db89 is described below
commit 3298d6db89ad5cd2cf0e73173088df317a4d1fe2
Author: Daniel Roberts
AuthorDate: Fri Feb 16 17:02:50 2024 -0500
Add Compaction Job Min & Max Wait properties (#4223)
* Add Compaction Min Wait and Max Wait properties
* Adds min and max wait properties to configure the min and max wait
intervals
in the compactor.
* Changes the logic in compaction-coordinator to use these new properties
when calculating the wait period for sending warning messages
* Also use the MAX_JOB_WAIT_TIME prop for the thrift retry interval when
the compactor is unable to communicate with the compaction-coordinator.
---
.../org/apache/accumulo/core/conf/Property.java| 10
.../coordinator/CompactionCoordinator.java | 4 +--
.../coordinator/CompactionCoordinatorTest.java | 21
.../org/apache/accumulo/compactor/Compactor.java | 21 ++--
.../apache/accumulo/compactor/CompactorTest.java | 29 ++
5 files changed, 75 insertions(+), 10 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 1fa04490fb..08a93ae6b3 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -1433,6 +1433,16 @@ public enum Property {
COMPACTOR_PREFIX("compactor.", null, PropertyType.PREFIX,
"Properties in this category affect the behavior of the accumulo
compactor server.", "2.1.0"),
@Experimental
+ COMPACTOR_MIN_JOB_WAIT_TIME("compactor.wait.time.job.min", "1s",
PropertyType.TIMEDURATION,
+ "The minimum amount of time to wait between checks for the next
compaction job, backing off"
+ + "exponentially until COMPACTOR_MAX_JOB_WAIT_TIME is reached.",
+ "2.1.3"),
+ @Experimental
+ COMPACTOR_MAX_JOB_WAIT_TIME("compactor.wait.time.job.max", "5m",
PropertyType.TIMEDURATION,
+ "Compactors do exponential backoff when their request for work
repeatedly come back empty. "
+ + "This is the maximum amount of time to wait between checks for the
next compaction job.",
+ "2.1.3"),
+ @Experimental
COMPACTOR_PORTSEARCH("compactor.port.search", "false", PropertyType.BOOLEAN,
"If the compactor.port.client is in use, search higher ports until one
is available.",
"2.1.0"),
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index df94ccb824..337f5bc685 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -101,8 +101,6 @@ public class CompactionCoordinator extends AbstractServer
private static final Logger LOG =
LoggerFactory.getLogger(CompactionCoordinator.class);
private static final long TIME_BETWEEN_GC_CHECKS = 5000;
- private static final long FIFTEEN_MINUTES = TimeUnit.MINUTES.toMillis(15);
-
protected static final QueueSummaries QUEUE_SUMMARIES = new QueueSummaries();
/*
@@ -383,7 +381,7 @@ public class CompactionCoordinator extends AbstractServer
}
protected long getMissingCompactorWarningTime() {
-return FIFTEEN_MINUTES;
+return
getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME) * 3;
}
protected long getTServerCheckInterval() {
diff --git
a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
index 7f46d68e7f..117d50108a 100644
---
a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
+++
b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
@@ -39,7 +39,10 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
import org.apache.accumulo.core.compaction.thrift.TExternalCompaction;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.DefaultConfigurat
(accumulo-examples) 01/01: Start point
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch helm-chart-collab
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
commit d5031384604bddec23b67843343665534acc8887
Author: Daniel Roberts
AuthorDate: Thu Feb 15 17:24:12 2024 +
Start point
---
helm/accumulo/.helmignore | 23 ++
helm/accumulo/Chart.yaml | 24 +++
helm/accumulo/templates/NOTES.txt | 22 ++
helm/accumulo/templates/_helpers.tpl | 62
helm/accumulo/templates/deployment.yaml| 61
helm/accumulo/templates/hpa.yaml | 28
helm/accumulo/templates/ingress.yaml | 61
helm/accumulo/templates/service.yaml | 15
helm/accumulo/templates/serviceaccount.yaml| 12
helm/accumulo/templates/tests/test-connection.yaml | 15
helm/accumulo/values.yaml | 82 ++
11 files changed, 405 insertions(+)
diff --git a/helm/accumulo/.helmignore b/helm/accumulo/.helmignore
new file mode 100644
index 000..0e8a0eb
--- /dev/null
+++ b/helm/accumulo/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/helm/accumulo/Chart.yaml b/helm/accumulo/Chart.yaml
new file mode 100644
index 000..163dd45
--- /dev/null
+++ b/helm/accumulo/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: accumulo
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into
versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart
developer. They're included as
+# a dependency of application charts to inject those utilities and functions
into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be
deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each
time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version
number should be
+# incremented each time you make changes to the application. Versions are not
expected to
+# follow Semantic Versioning. They should reflect the version the application
is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
diff --git a/helm/accumulo/templates/NOTES.txt
b/helm/accumulo/templates/NOTES.txt
new file mode 100644
index 000..71a6b4e
--- /dev/null
+++ b/helm/accumulo/templates/NOTES.txt
@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+ {{- range .paths }}
+ http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+ {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+ export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o
jsonpath="{.spec.ports[0].nodePort}" services {{ include "accumulo.fullname" .
}})
+ export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o
jsonpath="{.items[0].status.addresses[0].address}")
+ echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+ NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+ You can watch the status of by running 'kubectl get --namespace {{
.Release.Namespace }} svc -w {{ include "accumulo.fullname" . }}'
+ export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{
include "accumulo.fullname" . }} --template "{{"{{ range (index
.status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+ echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+ export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l
"app.kubernetes.io/name={{ include "accumulo.name" .
}},app.kubernetes.io/instance={{ .Release.Name }}" -o
jsonpath="{.items[0].metadata.name}")
+ export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }}
$POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+ echo "Visit h
(accumulo-examples) branch helm-chart-collab created (now d503138)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch helm-chart-collab in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git at d503138 Start point This branch includes the following new commits: new d503138 Start point The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(accumulo) branch elasticity updated (9d80b65f0e -> 0739f7ee6d)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9d80b65f0e fixes clean shutdown bug in manager (#4248) add a0c7e4ae36 Makes inuse GC deletion default behavior (#4252) new 0739f7ee6d Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/conf/Property.java| 4 - .../main/java/org/apache/accumulo/gc/GCRun.java| 10 --- .../accumulo/gc/GarbageCollectionAlgorithm.java| 4 +- .../accumulo/gc/GarbageCollectionEnvironment.java | 10 +-- .../apache/accumulo/gc/GarbageCollectionTest.java | 88 -- 5 files changed, 34 insertions(+), 82 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0739f7ee6d4db2931d855abefea00e70a32ada86 Merge: 9d80b65f0e a0c7e4ae36 Author: Daniel Roberts AuthorDate: Mon Feb 12 19:18:36 2024 + Merge branch 'main' into elasticity .../org/apache/accumulo/core/conf/Property.java| 4 - .../main/java/org/apache/accumulo/gc/GCRun.java| 10 --- .../accumulo/gc/GarbageCollectionAlgorithm.java| 4 +- .../accumulo/gc/GarbageCollectionEnvironment.java | 10 +-- .../apache/accumulo/gc/GarbageCollectionTest.java | 88 -- 5 files changed, 34 insertions(+), 82 deletions(-)
(accumulo) branch main updated: Makes inuse GC deletion default behavior (#4252)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a0c7e4ae36 Makes inuse GC deletion default behavior (#4252)
a0c7e4ae36 is described below
commit a0c7e4ae36c68df418103118fe0dde367b8bb94c
Author: Daniel Roberts
AuthorDate: Mon Feb 12 14:09:38 2024 -0500
Makes inuse GC deletion default behavior (#4252)
Removes the property allowing disabling of the in-use GC candidate
deletion feature
---
.../org/apache/accumulo/core/conf/Property.java| 4 -
.../main/java/org/apache/accumulo/gc/GCRun.java| 10 ---
.../accumulo/gc/GarbageCollectionAlgorithm.java| 4 +-
.../accumulo/gc/GarbageCollectionEnvironment.java | 10 +--
.../apache/accumulo/gc/GarbageCollectionTest.java | 88 --
5 files changed, 34 insertions(+), 82 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9acce283cd..497486cb18 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -794,10 +794,6 @@ public enum Property {
"The listening port for the garbage collector's monitor service.",
"1.3.5"),
GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT,
"The number of threads used to delete RFiles and write-ahead logs.",
"1.3.5"),
- GC_REMOVE_IN_USE_CANDIDATES("gc.remove.in.use.candidates", "true",
PropertyType.BOOLEAN,
- "GC will remove deletion candidates that are in-use from the metadata
location. "
- + "This is expected to increase the speed of subsequent GC runs.",
- "2.1.3"),
GC_SAFEMODE("gc.safemode", "false", PropertyType.BOOLEAN,
"Provides listing of files to be deleted but does not delete any
files.", "2.1.0"),
GC_USE_FULL_COMPACTION("gc.post.metadata.action", "flush",
PropertyType.GC_POST_ACTION,
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
index c210b58256..f02b85634e 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@ -472,16 +472,6 @@ public class GCRun implements GarbageCollectionEnvironment
{
return context.getConfiguration().getBoolean(Property.GC_SAFEMODE);
}
- /**
- * Checks if InUse Candidates can be removed.
- *
- * @return value of {@link Property#GC_REMOVE_IN_USE_CANDIDATES}
- */
- @Override
- public boolean canRemoveInUseCandidates() {
-return
context.getConfiguration().getBoolean(Property.GC_REMOVE_IN_USE_CANDIDATES);
- }
-
/**
* Moves a file to trash. If this garbage collector is not using trash, this
method returns false
* and leaves the file alone. If the file is missing, this method returns
false as opposed to
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index ab9bff706e..eb6b8fe281 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -198,9 +198,7 @@ public class GarbageCollectionAlgorithm {
Set tableIdsAfter = gce.getCandidateTableIDs();
ensureAllTablesChecked(Collections.unmodifiableSet(tableIdsBefore),
Collections.unmodifiableSet(tableIdsSeen),
Collections.unmodifiableSet(tableIdsAfter));
-if (gce.canRemoveInUseCandidates()) {
- gce.deleteGcCandidates(candidateEntriesToBeDeleted,
GcCandidateType.INUSE);
-}
+gce.deleteGcCandidates(candidateEntriesToBeDeleted, GcCandidateType.INUSE);
}
private long removeBlipCandidates(GarbageCollectionEnvironment gce,
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
index 1d2b1e6391..3c373e92b7 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
@@ -28,7 +28,6 @@ import java.util.SortedMap;
import java.util.stream.Stream;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.gc.GcCandidate;
import org.apache.accumulo.core.gc.Reference;
@@ -48,13 +47,6 @@ public interface GarbageCollectionEnvironment {
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 252b1e283394e0b0c77c192cbf9ad459c9a1a1d1 Merge: 878e4edfee 8bb8fcea89 Author: Daniel Roberts AuthorDate: Wed Feb 7 00:28:05 2024 + Merge branch 'main' into elasticity core/src/main/java/org/apache/accumulo/core/conf/Property.java | 1 - .../apache/accumulo/server/compaction/RetryableThriftCall.java | 9 - .../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 2 +- 3 files changed, 9 insertions(+), 3 deletions(-)
(accumulo) branch elasticity updated (878e4edfee -> 252b1e2833)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 878e4edfee Host tablets needing recovery (#4229) add 354be4fbaa Remove Experimental tag from GC property (#4222) add e27b8a3c53 Remove excessive logging message (#4237) add f98c87cb57 Improves thrift error logging level (#4236) add 8bb8fcea89 Merge branch '2.1' new 252b1e2833 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: core/src/main/java/org/apache/accumulo/core/conf/Property.java | 1 - .../apache/accumulo/server/compaction/RetryableThriftCall.java | 9 - .../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 2 +- 3 files changed, 9 insertions(+), 3 deletions(-)
(accumulo) branch main updated (354be4fbaa -> 8bb8fcea89)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 354be4fbaa Remove Experimental tag from GC property (#4222) add e27b8a3c53 Remove excessive logging message (#4237) add f98c87cb57 Improves thrift error logging level (#4236) new 8bb8fcea89 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/server/compaction/RetryableThriftCall.java | 9 - .../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 2 +- 2 files changed, 9 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8bb8fcea890a378608b101cb015fc921b684fde9 Merge: 354be4fbaa f98c87cb57 Author: Daniel Roberts AuthorDate: Wed Feb 7 00:20:23 2024 + Merge branch '2.1' .../apache/accumulo/server/compaction/RetryableThriftCall.java | 9 - .../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 2 +- 2 files changed, 9 insertions(+), 2 deletions(-)
(accumulo) branch 2.1 updated: Improves thrift error logging level (#4236)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new f98c87cb57 Improves thrift error logging level (#4236)
f98c87cb57 is described below
commit f98c87cb5798c781364a7773d17b1325a73b9f5d
Author: Daniel Roberts
AuthorDate: Tue Feb 6 19:03:33 2024 -0500
Improves thrift error logging level (#4236)
Drops thrift connection errors down to a `debug` level
On every 5th error message, the logger will elevate that message to
`warn` instead of `debug`.
---
.../apache/accumulo/server/compaction/RetryableThriftCall.java | 9 -
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java
index 6cfba2ee23..020e436af6 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java
@@ -95,11 +95,18 @@ public class RetryableThriftCall {
*/
public T run() throws RetriesExceededException {
T result = null;
+var errorsSeen = 0;
do {
try {
result = function.execute();
} catch (TException e) {
-LOG.error("Error in Thrift function, retrying ...", e);
+errorsSeen++;
+// Log higher levels of errors on every 5th error
+if (errorsSeen >= 5 && errorsSeen % 5 == 0) {
+ LOG.warn("Error in Thrift function, retrying ...", e);
+} else {
+ LOG.debug("Error in Thrift function, retrying ...", e);
+}
}
if (result == null) {
if (this.retry.canRetry()) {
(accumulo) branch 2.1 updated: Remove excessive logging message (#4237)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e27b8a3c53 Remove excessive logging message (#4237)
e27b8a3c53 is described below
commit e27b8a3c5304cd023a1194cf3770ed7b749d4f50
Author: Daniel Roberts
AuthorDate: Tue Feb 6 18:59:56 2024 -0500
Remove excessive logging message (#4237)
---
.../tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index a34c2b1adc..1425b7bcea 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@ -905,7 +905,7 @@ public class ScanServer extends AbstractServer
public ScanResult continueScan(TInfo tinfo, long scanID, long busyTimeout)
throws NoSuchScanIDException, NotServingTabletException,
TooManyFilesException,
TSampleNotPresentException, TException {
-LOG.debug("continue scan: {}", scanID);
+LOG.trace("continue scan: {}", scanID);
try (ScanReservation reservation = reserveFiles(scanID)) {
Preconditions.checkState(reservation.getFailures().isEmpty());
(accumulo) branch main updated: Remove Experimental tag from GC property (#4222)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 354be4fbaa Remove Experimental tag from GC property (#4222)
354be4fbaa is described below
commit 354be4fbaaa23df7a0377a35135dcd04eee18d4f
Author: Daniel Roberts
AuthorDate: Fri Feb 2 15:39:32 2024 -0500
Remove Experimental tag from GC property (#4222)
Since this property is enabled by default it should no longer be
considered `experimental` status.
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 90253af31c..56f856410d 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -792,7 +792,6 @@ public enum Property {
"The listening port for the garbage collector's monitor service.",
"1.3.5"),
GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT,
"The number of threads used to delete RFiles and write-ahead logs.",
"1.3.5"),
- @Experimental
GC_REMOVE_IN_USE_CANDIDATES("gc.remove.in.use.candidates", "true",
PropertyType.BOOLEAN,
"GC will remove deletion candidates that are in-use from the metadata
location. "
+ "This is expected to increase the speed of subsequent GC runs.",
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 37782d558e81115a63bc85e9ada9e6ddf51d11a9 Merge: 5fb05efd26 cdd86f18f2 Author: Daniel Roberts AuthorDate: Fri Feb 2 04:33:24 2024 + Merge branch '2.1'
(accumulo) branch main updated (5fb05efd26 -> 37782d558e)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5fb05efd26 Merge branch '2.1' add cdd86f18f2 Better error when compaction executors are not set (#4212) new 37782d558e Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch 2.1 updated: Better error when compaction executors are not set (#4212)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new cdd86f18f2 Better error when compaction executors are not set (#4212)
cdd86f18f2 is described below
commit cdd86f18f2cce1d00a21297bb927587cbd7747bf
Author: Daniel Roberts
AuthorDate: Thu Feb 1 23:24:53 2024 -0500
Better error when compaction executors are not set (#4212)
* Better error when compaction executors are not set
Replaces NPE with proper exception type and message for when the
Default Compaction Planner's "executors" property is set and empty
or not set at all.
---
.../spi/compaction/DefaultCompactionPlanner.java | 94 --
.../compaction/DefaultCompactionPlannerTest.java | 34
2 files changed, 84 insertions(+), 44 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index aae0591567..5a30556f63 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -167,62 +167,68 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
justification = "Field is written by Gson")
@Override
public void init(InitParameters params) {
-ExecutorConfig[] execConfigs =
-new Gson().fromJson(params.getOptions().get("executors"),
ExecutorConfig[].class);
-List tmpExec = new ArrayList<>();
+if (params.getOptions().containsKey("executors")
+&& !params.getOptions().get("executors").isBlank()) {
-for (ExecutorConfig executorConfig : execConfigs) {
- Long maxSize = executorConfig.maxSize == null ? null
- :
ConfigurationTypeHelper.getFixedMemoryAsBytes(executorConfig.maxSize);
+ ExecutorConfig[] execConfigs =
+ new Gson().fromJson(params.getOptions().get("executors"),
ExecutorConfig[].class);
- CompactionExecutorId ceid;
+ List tmpExec = new ArrayList<>();
- // If not supplied, GSON will leave type null. Default to internal
- if (executorConfig.type == null) {
-executorConfig.type = "internal";
- }
+ for (ExecutorConfig executorConfig : execConfigs) {
+Long maxSize = executorConfig.maxSize == null ? null
+:
ConfigurationTypeHelper.getFixedMemoryAsBytes(executorConfig.maxSize);
- switch (executorConfig.type) {
-case "internal":
- Preconditions.checkArgument(null == executorConfig.queue,
- "'queue' should not be specified for internal compactions");
- int numThreads = Objects.requireNonNull(executorConfig.numThreads,
- "'numThreads' must be specified for internal type");
- ceid =
params.getExecutorManager().createExecutor(executorConfig.name, numThreads);
- break;
-case "external":
- Preconditions.checkArgument(null == executorConfig.numThreads,
- "'numThreads' should not be specified for external compactions");
- String queue = Objects.requireNonNull(executorConfig.queue,
- "'queue' must be specified for external type");
- ceid = params.getExecutorManager().getExternalExecutor(queue);
- break;
-default:
- throw new IllegalArgumentException("type must be 'internal' or
'external'");
- }
- tmpExec.add(new Executor(ceid, maxSize));
-}
+CompactionExecutorId ceid;
-Collections.sort(tmpExec, Comparator.comparing(Executor::getMaxSize,
-Comparator.nullsLast(Comparator.naturalOrder(;
+// If not supplied, GSON will leave type null. Default to internal
+if (executorConfig.type == null) {
+ executorConfig.type = "internal";
+}
-executors = List.copyOf(tmpExec);
+switch (executorConfig.type) {
+ case "internal":
+Preconditions.checkArgument(null == executorConfig.queue,
+"'queue' should not be specified for internal compactions");
+int numThreads = Objects.requireNonNull(executorConfig.numThreads,
+"'numThreads' must be specified for internal type");
+ceid =
params.getExecutorManager().createExecutor(executorConfig.name, numThreads);
+break;
+ case "external":
+Preconditions.checkArgument(null == executorConfig.numThreads,
+"'numThreads' should not be specified for external
compactions
(accumulo) branch elasticity updated: Add comments (#4189)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e364ca6c54 Add comments (#4189)
e364ca6c54 is described below
commit e364ca6c54683c0ba4adadbc452daefccba93168
Author: Daniel Roberts
AuthorDate: Wed Jan 24 15:12:38 2024 -0500
Add comments (#4189)
Add additional comments for describing the goal of formatString
---
.../org/apache/accumulo/core/metrics/MetricsUtil.java | 19 ---
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
index 581c30f7eb..311bb5e190 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
@@ -142,29 +142,34 @@ public class MetricsUtil {
}
/**
- * Centralize any specific string formatting for metric names and/or tags.
Ensure strings match
- * the micrometer naming convention.
+ * This method replaces any intended delimiters with the "." delimiter that
is used by micrometer.
+ * Micrometer will then transform these delimiters to the metric producer's
delimiter. Example:
+ * "compactorQueue" becomes "compactor.queue" in micrometer. When using
Prometheus,
+ * "compactor.queue" would become "compactor_queue".
+ *
*/
public static String formatString(String name) {
-// Handle spaces
+// Replace spaces with dot delimiter
name = name.replace(" ", ".");
-// Handle snake_case notation
+// Replace snake_case with dot delimiter
name = name.replace("_", ".");
-// Handle Hyphens
+// Replace hyphens with dot delimiter
name = name.replace("-", ".");
-// Handle camelCase notation
+// Insert a dot delimiter before each capital letter found in the regex
pattern.
Matcher matcher = camelCasePattern.matcher(name);
StringBuilder output = new StringBuilder(name);
int insertCount = 0;
while (matcher.find()) {
- // Pattern matches at a lowercase letter, but the insert is at the
second position.
+ // Pattern matches on a "aAa" pattern and inserts the dot before the
uppercase character.
+ // Results in "aAa" becoming "a.Aa".
output.insert(matcher.start() + 1 + insertCount, ".");
// The correct index position will shift as inserts occur.
insertCount++;
}
name = output.toString();
+// remove all capital letters after the dot delimiters have been inserted.
return name.toLowerCase();
}
(accumulo) branch elasticity updated: Fixes Queue Metric Tag format (#4186)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c37f739eae Fixes Queue Metric Tag format (#4186)
c37f739eae is described below
commit c37f739eaea7425387c2cbb83307a3c39a0df1d1
Author: Daniel Roberts
AuthorDate: Wed Jan 24 10:42:55 2024 -0500
Fixes Queue Metric Tag format (#4186)
Adds back in the use of the MetricsUtil.formatString method when
creating metric tags
Removes missed "e." external queue name prefixes
---
.../manager/compaction/coordinator/QueueMetrics.java | 20 +---
.../compaction/CompactionPriorityQueueMetricsIT.java | 15 +++
.../test/compaction/ExternalCompactionMetricsIT.java | 2 +-
3 files changed, 21 insertions(+), 16 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
index f12135ba74..c5ba5157dc 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.manager.compaction.coordinator;
+import static org.apache.accumulo.core.metrics.MetricsUtil.formatString;
import static org.apache.accumulo.core.metrics.MetricsUtil.getCommonTags;
import java.util.HashMap;
@@ -51,40 +52,37 @@ public class QueueMetrics implements MetricsProducer {
private final Gauge jobsRejected;
private final Gauge jobsLowestPriority;
-public QueueMeters(MeterRegistry meterRegistry, CompactorGroupId queueId,
+public QueueMeters(MeterRegistry meterRegistry, CompactorGroupId cgid,
CompactionJobPriorityQueue queue) {
+ var queueId = formatString(cgid.canonical());
+
length =
Gauge.builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH, queue, q
-> q.getMaxSize())
.description("Length of priority queues")
- .tags(Tags.concat(getCommonTags(), "queue.id",
queueId.canonical()))
- .register(meterRegistry);
+ .tags(Tags.concat(getCommonTags(), "queue.id",
queueId)).register(meterRegistry);
jobsQueued = Gauge
.builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED, queue, q
-> q.getQueuedJobs())
.description("Count of queued jobs")
- .tags(Tags.concat(getCommonTags(), "queue.id", queueId.canonical()))
- .register(meterRegistry);
+ .tags(Tags.concat(getCommonTags(), "queue.id",
queueId)).register(meterRegistry);
jobsDequeued = Gauge
.builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED, queue,
q -> q.getDequeuedJobs())
.description("Count of jobs dequeued")
- .tags(Tags.concat(getCommonTags(), "queue.id", queueId.canonical()))
- .register(meterRegistry);
+ .tags(Tags.concat(getCommonTags(), "queue.id",
queueId)).register(meterRegistry);
jobsRejected = Gauge
.builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED, queue,
q -> q.getRejectedJobs())
.description("Count of rejected jobs")
- .tags(Tags.concat(getCommonTags(), "queue.id", queueId.canonical()))
- .register(meterRegistry);
+ .tags(Tags.concat(getCommonTags(), "queue.id",
queueId)).register(meterRegistry);
jobsLowestPriority = Gauge
.builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY, queue,
q -> q.getLowestPriority())
.description("Lowest priority queued job")
- .tags(Tags.concat(getCommonTags(), "queue.id", queueId.canonical()))
- .register(meterRegistry);
+ .tags(Tags.concat(getCommonTags(), "queue.id",
queueId)).register(meterRegistry);
}
private void removeMeters(MeterRegistry registry) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
index f56dfd668f..e89f96dfaa 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionPriorityQueueMetricsIT.java
@@ -56,10 +56,12 @@ import
org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsProducer;
import
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f444b85f5d6c4fae028b45ecd4a54fb11e1f766c Merge: e468fd30b2 731bbf75e7 Author: Daniel Roberts AuthorDate: Mon Jan 22 19:21:44 2024 + Merge branch 'main' into elasticity .../java/org/apache/accumulo/core/logging/TabletLogger.java | 13 + 1 file changed, 13 insertions(+)
(accumulo) branch elasticity updated (e468fd30b2 -> f444b85f5d)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from e468fd30b2 Merge branch 'main' into elasticity add 731bbf75e7 Compaction Failed Logging in TabletLogger (#4153) new f444b85f5d Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/logging/TabletLogger.java | 13 + 1 file changed, 13 insertions(+)
(accumulo) branch main updated: Compaction Failed Logging in TabletLogger (#4153)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 731bbf75e7 Compaction Failed Logging in TabletLogger (#4153)
731bbf75e7 is described below
commit 731bbf75e75c38247d13d810de986b2e15659ff6
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Mon Jan 22 13:21:42 2024 -0500
Compaction Failed Logging in TabletLogger (#4153)
Closes #4139
Changes:
- Methods compactionFailed() and externalCompactionFailed() in TabletLogger
- Small change to arg passed to updateTimer()
- Calls to compactionFailed() and externalCompactionFailed() in
CompactableImpl
---
.../java/org/apache/accumulo/core/logging/TabletLogger.java | 13 +
.../org/apache/accumulo/tserver/tablet/CompactableImpl.java | 5 -
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
index 44832db7f1..d01356cfc7 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.core.metadata.CompactableFileImpl;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
@@ -148,6 +149,18 @@ public class TabletLogger {
asMinimalString(job.getFiles()));
}
+ public static void compactionFailed(KeyExtent extent, CompactionJob job,
+ CompactionConfig config) {
+fileLog.debug("Failed to compact: extent: {}, input files: {}, iterators:
{}", extent,
+asMinimalString(job.getFiles()), config.getIterators());
+ }
+
+ public static void externalCompactionFailed(KeyExtent extent,
ExternalCompactionId id,
+ CompactionJob job, CompactionConfig config) {
+fileLog.debug("Failed to compact: id: {}, extent: {}, input files: {},
iterators: {}", id,
+extent, asMinimalString(job.getFiles()), config.getIterators());
+ }
+
public static void flushed(KeyExtent extent, Optional
newDatafile) {
if (newDatafile.isPresent()) {
fileLog.debug("Flushed {} created {} from [memory]", extent,
newDatafile.orElseThrow());
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 6aa7401b14..28d6b11977 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -1108,10 +1108,11 @@ public class CompactableImpl implements Compactable {
log.debug("Compaction canceled {} ", getExtent());
} catch (Exception e) {
newFile = Optional.empty();
+ TabletLogger.compactionFailed(getExtent(), job,
cInfo.localCompactionCfg);
throw new RuntimeException(e);
} finally {
completeCompaction(job, cInfo.jobFiles, newFile, successful);
- tablet.updateTimer(MAJOR, queuedTime, startTime, stats.getEntriesRead(),
newFile == null);
+ tablet.updateTimer(MAJOR, queuedTime, startTime, stats.getEntriesRead(),
newFile.isEmpty());
}
}
@@ -1161,6 +1162,8 @@ public class CompactableImpl implements Compactable {
} catch (Exception e) {
externalCompactions.remove(externalCompactionId);
+ TabletLogger.externalCompactionFailed(getExtent(), externalCompactionId,
job,
+ cInfo.localCompactionCfg);
completeCompaction(job, cInfo.jobFiles, Optional.empty(), false);
throw new RuntimeException(e);
}
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e468fd30b2981133a9dc131672e9fbc4f1ba5aa0 Merge: fd387ba605 5922d90b95 Author: Daniel Roberts AuthorDate: Mon Jan 22 17:06:11 2024 + Merge branch 'main' into elasticity assemble/src/main/resources/LICENSE| 42 +++--- .../accumulo/server/compaction/FileCompactor.java | 16 +++-- .../org/apache/accumulo/compactor/Compactor.java | 11 -- .../coordinator/CompactionCoordinator.java | 3 +- .../coordinator/DeadCompactionDetector.java| 4 +-- .../compaction/ExternalDoNothingCompactor.java | 4 ++- 6 files changed, 49 insertions(+), 31 deletions(-) diff --cc server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 3deae9424a,00..283d2c1327 mode 100644,00..100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@@ -1,1003 -1,0 +1,1004 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.compaction.coordinator; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.TableDeletedException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.CompactionConfig; +import org.apache.accumulo.core.client.admin.compaction.CompactableFile; +import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; +import org.apache.accumulo.core.clientImpl.thrift.TInfo; +import org.apache.accumulo.core.clientImpl.thrift.TableOperation; +import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType; +import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; +import org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException; +import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; +import org.apache.accumulo.core.compaction.thrift.TCompactionState; +import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate; +import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; +import org.apache.accumulo.core.compaction.thrift.TExternalCompactionList; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.NamespaceId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.dataImpl.thrift.
(accumulo) branch elasticity updated (fd387ba605 -> e468fd30b2)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from fd387ba605 Merge branch 'main' into elasticity add fcaabbe427 More Detailed Compaction Errors (#4148) add c0d548aae3 Fix assembly LICENSE formatting add 5922d90b95 Merge branch '2.1' new e468fd30b2 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/src/main/resources/LICENSE| 42 +++--- .../accumulo/server/compaction/FileCompactor.java | 16 +++-- .../org/apache/accumulo/compactor/Compactor.java | 11 -- .../coordinator/CompactionCoordinator.java | 3 +- .../coordinator/DeadCompactionDetector.java| 4 +-- .../compaction/ExternalDoNothingCompactor.java | 4 ++- 6 files changed, 49 insertions(+), 31 deletions(-)
(accumulo) branch main updated (bed5e3628d -> 5922d90b95)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from bed5e3628d Merge branch '2.1' add fcaabbe427 More Detailed Compaction Errors (#4148) add c0d548aae3 Fix assembly LICENSE formatting new 5922d90b95 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/src/main/resources/LICENSE| 42 +++--- .../accumulo/server/compaction/FileCompactor.java | 16 +++-- .../coordinator/CompactionCoordinator.java | 3 +- .../accumulo/coordinator/CompactionFinalizer.java | 3 +- .../coordinator/DeadCompactionDetector.java| 4 +-- .../org/apache/accumulo/compactor/Compactor.java | 11 -- .../accumulo/tserver/tablet/CompactableImpl.java | 5 +-- .../compaction/ExternalDoNothingCompactor.java | 5 ++- 8 files changed, 55 insertions(+), 34 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5922d90b95bcd611ed77515f4097af2473b0519b Merge: bed5e3628d c0d548aae3 Author: Daniel Roberts AuthorDate: Mon Jan 22 16:44:42 2024 + Merge branch '2.1' assemble/src/main/resources/LICENSE| 42 +++--- .../accumulo/server/compaction/FileCompactor.java | 16 +++-- .../coordinator/CompactionCoordinator.java | 3 +- .../accumulo/coordinator/CompactionFinalizer.java | 3 +- .../coordinator/DeadCompactionDetector.java| 4 +-- .../org/apache/accumulo/compactor/Compactor.java | 11 -- .../accumulo/tserver/tablet/CompactableImpl.java | 5 +-- .../compaction/ExternalDoNothingCompactor.java | 5 ++- 8 files changed, 55 insertions(+), 34 deletions(-)
(accumulo) branch 2.1 updated: More Detailed Compaction Errors (#4148)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new fcaabbe427 More Detailed Compaction Errors (#4148)
fcaabbe427 is described below
commit fcaabbe427177eabf8f3dc24a29fb668677eb86e
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Mon Jan 22 11:07:52 2024 -0500
More Detailed Compaction Errors (#4148)
* More Detailed Compaction Errors
Closes #4075
Changes:
- CompactionFinalizer
- Added compactionId to log of compaction failure (tableId
is already included from 'ecfs')
- CompactionCoordinator
- Added extent/tableId to log of compaction failure
(compactionId already included)
- CompactableImpl
- Added extent/tableId to two logs of compaction failure
(compactionId already included)
- Compactor
- Added extent/tableId and compactionId to three of the logs
- ExternalDoNothingCompactor
- Added extent/tableId and compactionId to log of
compaction failure
- DeadCompactionDetector
- Added extent/tableId to log (compactionId already
included)
- FileCompactor
- Added extent/tableId, input files, output file,
iterators, and thread start time to log when error occurs
---
.../apache/accumulo/server/compaction/FileCompactor.java | 16 +---
.../accumulo/coordinator/CompactionCoordinator.java | 3 ++-
.../apache/accumulo/coordinator/CompactionFinalizer.java | 3 ++-
.../accumulo/coordinator/DeadCompactionDetector.java | 4 ++--
.../java/org/apache/accumulo/compactor/Compactor.java| 11 ---
.../org/apache/accumulo/compactor/CompactorTest.java | 5 +++--
.../apache/accumulo/tserver/tablet/CompactableImpl.java | 5 +++--
.../test/compaction/ExternalDoNothingCompactor.java | 5 -
8 files changed, 37 insertions(+), 15 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
index 27eef8def0..7c16351ccf 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
@@ -75,6 +75,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Collections2;
+
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.context.Scope;
@@ -207,11 +209,13 @@ public class FileCompactor implements
Callable {
boolean remove = runningCompactions.add(this);
+String threadStartDate = dateFormatter.format(new Date());
+
clearStats();
String oldThreadName = Thread.currentThread().getName();
-String newThreadName = "MajC compacting " + extent + " started "
-+ dateFormatter.format(new Date()) + " file: " + outputFile;
+String newThreadName =
+"MajC compacting " + extent + " started " + threadStartDate + " file:
" + outputFile;
Thread.currentThread().setName(newThreadName);
thread = Thread.currentThread();
try {
@@ -279,7 +283,13 @@ public class FileCompactor implements
Callable {
log.debug("Compaction canceled {}", extent);
throw e;
} catch (IOException | RuntimeException e) {
- log.error("{}", e.getMessage(), e);
+ Collection inputFileNames =
+ Collections2.transform(getFilesToCompact(),
StoredTabletFile::getFileName);
+ String outputFileName = outputFile.getFileName();
+ log.error(
+ "Compaction error. Compaction info: "
+ + "extent: {}, input files: {}, output file: {}, iterators: {},
start date: {}",
+ getExtent(), inputFileNames, outputFileName, getIterators(),
threadStartDate, e);
throw e;
} finally {
Thread.currentThread().setName(oldThreadName);
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 88df289a56..df94ccb824 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -537,7 +537,8 @@ public class CompactionCoordinator extends AbstractServer
throw new AccumuloSecurityException(credentials.getPrincipal(),
SecurityErrorCode.PERM
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit fd387ba605613dd2725fe0de8a4eb4ff22e881ed
Merge: bd7d82e30b bed5e3628d
Author: Daniel Roberts
AuthorDate: Mon Jan 22 15:26:40 2024 +
Merge branch 'main' into elasticity
assemble/conf/accumulo-env.sh | 2 +-
assemble/src/main/resources/LICENSE| 42 +++---
pom.xml| 17 -
.../accumulo/server/tables/TableManager.java | 22 ++--
.../apache/accumulo/gc/SimpleGarbageCollector.java | 2 +-
.../accumulo/manager/FateServiceHandler.java | 16 ++---
.../manager/tableOps/ChangeTableState.java | 9 +++--
.../manager/tableOps/clone/FinishCloneTable.java | 8 +++--
.../manager/tableOps/create/FinishCreateTable.java | 6 ++--
.../manager/tableOps/delete/DeleteTable.java | 6 +++-
.../tableOps/tableImport/FinishImportTable.java| 5 ++-
11 files changed, 82 insertions(+), 53 deletions(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
index d8567ca2a2,9d66cdcc93..abef59d713
---
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
@@@ -66,10 -62,8 +67,11 @@@ import org.apache.accumulo.core.clientI
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.dataImpl.thrift.TRange;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus;
+ import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.manager.thrift.BulkImportState;
import org.apache.accumulo.core.manager.thrift.FateOperation;
import org.apache.accumulo.core.manager.thrift.FateService;
@@@ -407,9 -390,12 +409,12 @@@ class FateServiceHandler implements Fat
}
goalMessage += "Online table " + tableId;
+ final EnumSet expectedCurrStates =
+ EnumSet.of(TableState.ONLINE, TableState.OFFLINE);
-manager.fate().seedTransaction(op.toString(), opid,
+manager.fate(type).seedTransaction(op.toString(), tid,
- new TraceRepo<>(new ChangeTableState(namespaceId, tableId,
tableOp)), autoCleanup,
- goalMessage);
+ new TraceRepo<>(
+ new ChangeTableState(namespaceId, tableId, tableOp,
expectedCurrStates)),
+ autoCleanup, goalMessage);
break;
}
case TABLE_OFFLINE: {
@@@ -432,9 -418,12 +437,12 @@@
}
goalMessage += "Offline table " + tableId;
+ final EnumSet expectedCurrStates =
+ EnumSet.of(TableState.ONLINE, TableState.OFFLINE);
-manager.fate().seedTransaction(op.toString(), opid,
+manager.fate(type).seedTransaction(op.toString(), tid,
- new TraceRepo<>(new ChangeTableState(namespaceId, tableId,
tableOp)), autoCleanup,
- goalMessage);
+ new TraceRepo<>(
+ new ChangeTableState(namespaceId, tableId, tableOp,
expectedCurrStates)),
+ autoCleanup, goalMessage);
break;
}
case TABLE_MERGE: {
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/clone/FinishCloneTable.java
index 3733271049,9c2b46f2a2..b406d43243
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/clone/FinishCloneTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/clone/FinishCloneTable.java
@@@ -47,12 -49,13 +49,14 @@@ class FinishCloneTable extends ManagerR
// may never create files.. therefore there is no need to consume
namenode space w/ directories
// that are not used... tablet will create directories as needed
+ final EnumSet expectedCurrStates = EnumSet.of(TableState.NEW);
if (cloneInfo.keepOffline) {
- environment.getTableManager().transitionTableState(cloneInfo.tableId,
TableState.OFFLINE);
+ environment.getTableManager().transitionTableState(cloneInfo.tableId,
TableState.OFFLINE,
+ expectedCurrStates);
} else {
- environment.getTableManager().transitionTableState(cloneInfo.tableId,
TableState.ONLINE,
- expectedCurrStates);
+ // transition clone table state to state of original table
+ TableState ts =
environment.getTableManager().getTableState(cloneInfo.srcTableId);
- environment.getTableManager().transitionTableState(cloneInfo.tableId,
ts);
++ enviro
(accumulo) branch elasticity updated (bd7d82e30b -> fd387ba605)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from bd7d82e30b Add a default limit of deferred FATE transactions (#4179) add cf9417ad91 Warn when tablet needs compacting and is not (#4143) add 3bc0cfbfba Merge commit 'cf9417ad911b1f055d22620efd58238d1b0df236' add e67cff2db2 fixes split recovery for metadata tablet (#4137) add 0525755e2e Merge commit 'e67cff2db2a89ec203e96ad0b65183764c0f4342' add cefa3a405d Stricter checks before transitioning table state (#4165) add 33b889a98e Trivial javadoc update in SimpleGarbageCollector.isSafeMode. (#4174) add 68c7e20062 Merge branch '2.1' add 68874534bb Update various dependency versions (#4159) add 9a5a24fec2 Merge remote-tracking branch 'upstream/2.1' add 0071ee070f Fix paste invocation to work with Mac OS X (#4152) add bed5e3628d Merge branch '2.1' new fd387ba605 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/conf/accumulo-env.sh | 2 +- assemble/src/main/resources/LICENSE| 42 +++--- pom.xml| 17 - .../accumulo/server/tables/TableManager.java | 22 ++-- .../apache/accumulo/gc/SimpleGarbageCollector.java | 2 +- .../accumulo/manager/FateServiceHandler.java | 16 ++--- .../manager/tableOps/ChangeTableState.java | 9 +++-- .../manager/tableOps/clone/FinishCloneTable.java | 8 +++-- .../manager/tableOps/create/FinishCreateTable.java | 6 ++-- .../manager/tableOps/delete/DeleteTable.java | 6 +++- .../tableOps/tableImport/FinishImportTable.java| 5 ++- 11 files changed, 82 insertions(+), 53 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bed5e3628d9777d2e84587cced2fe9d9989accba Merge: 9a5a24fec2 0071ee070f Author: Daniel Roberts AuthorDate: Mon Jan 22 14:48:34 2024 + Merge branch '2.1' assemble/conf/accumulo-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch main updated (9a5a24fec2 -> bed5e3628d)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9a5a24fec2 Merge remote-tracking branch 'upstream/2.1' add 0071ee070f Fix paste invocation to work with Mac OS X (#4152) new bed5e3628d Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/conf/accumulo-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch 2.1 updated: Fix paste invocation to work with Mac OS X (#4152)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 0071ee070f Fix paste invocation to work with Mac OS X (#4152)
0071ee070f is described below
commit 0071ee070f4bcb24f1de05c48b13ef69f8e03ff5
Author: kristapsdz-saic <99488923+kristapsdz-s...@users.noreply.github.com>
AuthorDate: Mon Jan 22 06:46:31 2024 -0800
Fix paste invocation to work with Mac OS X (#4152)
Changed from original to be off 2.1.
References #4141
---
assemble/conf/accumulo-env.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/assemble/conf/accumulo-env.sh b/assemble/conf/accumulo-env.sh
index b87402fe9b..bcde6907be 100644
--- a/assemble/conf/accumulo-env.sh
+++ b/assemble/conf/accumulo-env.sh
@@ -61,7 +61,7 @@ if [[ -n $CLASSPATH ]]; then
else
CLASSPATH="${conf}"
fi
-ZK_JARS=$(find "$ZOOKEEPER_HOME/lib/" -maxdepth 1 -name '*.jar' -not -name
'*slf4j*' -not -name '*log4j*' | paste -sd:)
+ZK_JARS=$(find "$ZOOKEEPER_HOME/lib/" -maxdepth 1 -name '*.jar' -not -name
'*slf4j*' -not -name '*log4j*' | paste -sd: -)
# lib is set by calling script that sources this env file
#shellcheck disable=SC2154
CLASSPATH="${CLASSPATH}:${lib}/*:${HADOOP_CONF_DIR}:${ZOOKEEPER_HOME}/*:${ZK_JARS}:${HADOOP_HOME}/share/hadoop/client/*"
(accumulo) branch main updated: Switch from Guava to Caffeine (#4147)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7de3e0aa36 Switch from Guava to Caffeine (#4147)
7de3e0aa36 is described below
commit 7de3e0aa36d4792a259035231087bd26c0ba393b
Author: Daniel Roberts
AuthorDate: Wed Jan 10 11:07:24 2024 -0500
Switch from Guava to Caffeine (#4147)
Fixes 2.1 merge to main
---
.../org/apache/accumulo/tserver/compactions/CompactionService.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
index 0e18f17df3..c4f37bf9c4 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
@@ -67,9 +67,9 @@ import
org.apache.accumulo.tserver.metrics.CompactionExecutorsMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Preconditions;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
public class CompactionService {
@@ -140,7 +140,7 @@ public class CompactionService {
queuedForPlanning.put(kind, new
ConcurrentHashMap());
}
-maxScanFilesExceededErrorCache =
CacheBuilder.newBuilder().expireAfterWrite(5, MINUTES).build();
+maxScanFilesExceededErrorCache = Caffeine.newBuilder().expireAfterWrite(5,
MINUTES).build();
log.debug("Created new compaction service id:{} rate limit:{} planner:{}
planner options:{}",
myId, maxRate, plannerClass, plannerOptions);
(accumulo) branch main updated: Warn when tablet needs compacting and is not (#4143)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new f2c779ea56 Warn when tablet needs compacting and is not (#4143)
f2c779ea56 is described below
commit f2c779ea561dcc5b42b00ccb3afddfe5ded51bb0
Author: Daniel Roberts
AuthorDate: Tue Jan 9 15:26:05 2024 -0500
Warn when tablet needs compacting and is not (#4143)
* Warn when tablet needs compacting and is not
* Adds error logging suppression
Adds a 5 minute suppression for the empty compaction plan error message
that occures when tablet files exceed TSERV_SCAN_MAX_OPEN_FILES.
-
Co-authored-by: Keith Turner
---
.../tserver/compactions/CompactionService.java | 31 ++
1 file changed, 31 insertions(+)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
index d696c7f826..0e18f17df3 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.tserver.compactions;
+import static java.util.concurrent.TimeUnit.MINUTES;
+
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
@@ -40,6 +42,7 @@ import java.util.function.Function;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
@@ -65,6 +68,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
public class CompactionService {
@@ -84,6 +89,9 @@ public class CompactionService {
private AtomicLong rateLimit = new AtomicLong(0);
private Function
externExecutorSupplier;
+ // use to limit logging of max scan files exceeded
+ private final Cache maxScanFilesExceededErrorCache;
+
private static final Logger log =
LoggerFactory.getLogger(CompactionService.class);
public CompactionService(String serviceName, String plannerClass, String
plannerPrefix,
@@ -132,6 +140,8 @@ public class CompactionService {
queuedForPlanning.put(kind, new
ConcurrentHashMap());
}
+maxScanFilesExceededErrorCache =
CacheBuilder.newBuilder().expireAfterWrite(5, MINUTES).build();
+
log.debug("Created new compaction service id:{} rate limit:{} planner:{}
planner options:{}",
myId, maxRate, plannerClass, plannerOptions);
}
@@ -283,6 +293,27 @@ public class CompactionService {
CompactionPlan plan;
try {
plan = planner.makePlan(params);
+ var tableId = compactable.getTableId();
+
+ if (plan.getJobs().isEmpty()) {
+int maxScanFiles =
+
context.getTableConfiguration(tableId).getCount(Property.TSERV_SCAN_MAX_OPENFILES);
+
+if (files.allFiles.size() >= maxScanFiles &&
files.compacting.isEmpty()) {
+ var last = maxScanFilesExceededErrorCache.getIfPresent(tableId);
+
+ if (last == null) {
+log.warn(
+"The tablet {} has {} files and the max files for scan is {}.
No compactions are "
++ "running and none were planned for this tablet by {}, so
the files will "
++ "not be reduced by compaction which could cause scans to
fail. Please "
++ "check your compaction configuration. This log message
is temporarily suppressed for the entire table.",
+compactable.getExtent(), files.allFiles.size(), maxScanFiles,
myId);
+maxScanFilesExceededErrorCache.put(tableId,
System.currentTimeMillis());
+ }
+}
+ }
+
} catch (RuntimeException e) {
log.debug("Planner failed {} {} {} {} {}", myId,
planner.getClass().getName(),
compactable.getExtent(), kind, files, e);
(accumulo) branch 2.1 updated: Warn when tablet needs compacting and is not (#4143)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new cf9417ad91 Warn when tablet needs compacting and is not (#4143)
cf9417ad91 is described below
commit cf9417ad911b1f055d22620efd58238d1b0df236
Author: Daniel Roberts
AuthorDate: Tue Jan 9 15:26:05 2024 -0500
Warn when tablet needs compacting and is not (#4143)
* Warn when tablet needs compacting and is not
* Adds error logging suppression
Adds a 5 minute suppression for the empty compaction plan error message
that occures when tablet files exceed TSERV_SCAN_MAX_OPEN_FILES.
-
Co-authored-by: Keith Turner
---
.../tserver/compactions/CompactionService.java | 31 ++
1 file changed, 31 insertions(+)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
index 3dd2e32b36..e7bc18403d 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.tserver.compactions;
+import static java.util.concurrent.TimeUnit.MINUTES;
+
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
@@ -40,6 +42,7 @@ import java.util.function.Function;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
@@ -65,6 +68,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
public class CompactionService {
@@ -84,6 +89,9 @@ public class CompactionService {
private AtomicLong rateLimit = new AtomicLong(0);
private Function
externExecutorSupplier;
+ // use to limit logging of max scan files exceeded
+ private final Cache maxScanFilesExceededErrorCache;
+
private static final Logger log =
LoggerFactory.getLogger(CompactionService.class);
public CompactionService(String serviceName, String plannerClass, Long
maxRate,
@@ -132,6 +140,8 @@ public class CompactionService {
queuedForPlanning.put(kind, new
ConcurrentHashMap());
}
+maxScanFilesExceededErrorCache =
CacheBuilder.newBuilder().expireAfterWrite(5, MINUTES).build();
+
log.debug("Created new compaction service id:{} rate limit:{} planner:{}
planner options:{}",
myId, maxRate, plannerClass, plannerOptions);
}
@@ -283,6 +293,27 @@ public class CompactionService {
CompactionPlan plan;
try {
plan = planner.makePlan(params);
+ var tableId = compactable.getTableId();
+
+ if (plan.getJobs().isEmpty()) {
+int maxScanFiles =
+
context.getTableConfiguration(tableId).getCount(Property.TSERV_SCAN_MAX_OPENFILES);
+
+if (files.allFiles.size() >= maxScanFiles &&
files.compacting.isEmpty()) {
+ var last = maxScanFilesExceededErrorCache.getIfPresent(tableId);
+
+ if (last == null) {
+log.warn(
+"The tablet {} has {} files and the max files for scan is {}.
No compactions are "
++ "running and none were planned for this tablet by {}, so
the files will "
++ "not be reduced by compaction which could cause scans to
fail. Please "
++ "check your compaction configuration. This log message
is temporarily suppressed for the entire table.",
+compactable.getExtent(), files.allFiles.size(), maxScanFiles,
myId);
+maxScanFilesExceededErrorCache.put(tableId,
System.currentTimeMillis());
+ }
+}
+ }
+
} catch (RuntimeException e) {
log.debug("Planner failed {} {} {} {} {}", myId,
planner.getClass().getName(),
compactable.getExtent(), kind, files, e);
(accumulo-testing) branch elasticity updated (a97f1f0 -> 89b15be)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git from a97f1f0 Fix formatting of cluster.yaml.tftpl file add 89b15be Fix compaction service cs1 No new revisions were added by this update. Summary of changes: conf/accumulo-testing.properties | 8 1 file changed, 4 insertions(+), 4 deletions(-)
(accumulo) branch elasticity updated (fe8db50bc2 -> 9aabd60ef5)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from fe8db50bc2 Added metric that indicates when process is idle (#4078) add 319ec30b7f Adds compaction metadata merge test (#4119) add dc2800691e Merge branch '2.1' add 03934832f5 fix merge from 2.1 new 9aabd60ef5 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/test/functional/MergeIT.java | 59 ++ 1 file changed, 59 insertions(+)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9aabd60ef55d8054389f5c6c6a4833f86e796af4
Merge: fe8db50bc2 03934832f5
Author: Daniel Roberts
AuthorDate: Thu Jan 4 05:02:31 2024 +
Merge branch 'main' into elasticity
.../apache/accumulo/test/functional/MergeIT.java | 59 ++
1 file changed, 59 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index cf37fb3369,f63a2a99cd..856b778f19
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@@ -23,7 -23,7 +23,8 @@@ import static org.apache.accumulo.test.
import static
org.apache.accumulo.test.util.FileMetadataUtil.verifyMergedMarkerCleared;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@@ -34,10 -34,10 +35,12 @@@ import java.util.HashSet
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+ import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@@ -55,14 -53,18 +58,21 @@@ import org.apache.accumulo.core.data.Mu
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+ import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
+ import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
+ import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.core.util.Merge;
+ import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.TestIngest.IngestParams;
@@@ -662,4 -579,53 +672,53 @@@ public class MergeIT extends AccumuloCl
}
}
}
+
+ // Test that merge handles metadata from compactions
+ @Test
+ public void testCompactionMetadata() throws Exception {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ var split = new Text("m");
+ c.tableOperations().addSplits(tableName, new TreeSet<>(List.of(split)));
+
+ TableId tableId = getServerContext().getTableId(tableName);
+
+ // add metadata from compactions to tablets prior to merge
+ try (var tabletsMutator =
getServerContext().getAmple().mutateTablets()) {
+ for (var extent : List.of(new KeyExtent(tableId, split, null),
+ new KeyExtent(tableId, null, split))) {
+ var tablet = tabletsMutator.mutateTablet(extent);
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+
+ ReferencedTabletFile tmpFile =
+ ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
+ CompactionExecutorId ceid =
CompactionExecutorIdImpl.externalId("G1");
+ Set jobFiles =
+ Set.of(StoredTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/b2.rf")));
- ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, jobFiles,
- tmpFile, "localhost:", CompactionKind.SYSTEM, (short) 2,
ceid, false, false, 44L);
++ ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, tmpFile,
++ "localhost:", CompactionKind.SYSTEM, (short) 2, ceid,
false, 44L);
+ tablet.putExternalCompaction(ecid, ecMeta);
+
(accumulo) branch main updated: fix merge from 2.1
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 03934832f5 fix merge from 2.1
03934832f5 is described below
commit 03934832f5f153d3f5f8876307ecd4040dc3d0c3
Author: Daniel Roberts
AuthorDate: Thu Jan 4 04:44:29 2024 +
fix merge from 2.1
---
test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index af02e42f51..f63a2a99cd 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@ -603,7 +603,7 @@ public class MergeIT extends AccumuloClusterHarness {
ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
CompactionExecutorId ceid =
CompactionExecutorIdImpl.externalId("G1");
Set jobFiles =
- Set.of(new
StoredTabletFile("file:///accumulo/tables/t-0/b-0/b2.rf"));
+ Set.of(StoredTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/b2.rf")));
ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, jobFiles,
tmpFile, "localhost:", CompactionKind.SYSTEM, (short) 2,
ceid, false, false, 44L);
tablet.putExternalCompaction(ecid, ecMeta);
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit dc2800691e3b4e269d8f86d6b9f99861c4d9257d
Merge: 3792630c86 319ec30b7f
Author: Daniel Roberts
AuthorDate: Thu Jan 4 01:53:51 2024 +
Merge branch '2.1'
.../apache/accumulo/test/functional/MergeIT.java | 60 ++
1 file changed, 60 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index 4d265a1120,43590e8030..af02e42f51
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@@ -18,52 -18,46 +18,63 @@@
*/
package org.apache.accumulo.test.functional;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.accumulo.test.util.FileMetadataUtil.printAndVerifyFileMetadata;
+import static
org.apache.accumulo.test.util.FileMetadataUtil.verifyMergedMarkerCleared;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+ import java.util.UUID;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.dataImpl.KeyExtent;
++import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
import org.apache.accumulo.core.security.Authorizations;
+ import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
+ import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.core.util.Merge;
+ import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.TestIngest.IngestParams;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.accumulo.test.VerifyIngest.VerifyParams;
+ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MergeIT extends AccumuloClusterHarness {
@@@ -568,4 -230,52 +579,53 @@@
}
}
}
+
+ // Test that merge handles metadata from compactions
+ @Test
+ public void testCompactionMetadata() throws Exception {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ var split = new Text("m");
+ c.tableOperations().addSplits(tableName, new TreeSet<>(List.of(split)));
+
+ TableId tableId = getServerContext().getTableId(tableName);
+
+ // add metadata from compactions to tablets prior to merge
+ try (var tabletsMutator =
getServerContext().getAmple().mutateTablets()) {
+ for (var extent : List.of(new KeyExtent(tableId, split, null),
+ new KeyExtent(tableId, null, split))) {
+ var tablet = tabletsMutator.mutateTablet(extent);
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+
- TabletFile tmpFile = new TabletFile(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
++ ReferencedTabletFile tmpFile =
++ ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"
(accumulo) branch main updated (3792630c86 -> dc2800691e)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 3792630c86 Merge branch '2.1' add 319ec30b7f Adds compaction metadata merge test (#4119) new dc2800691e Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/test/functional/MergeIT.java | 60 ++ 1 file changed, 60 insertions(+)
(accumulo) branch 2.1 updated: Adds compaction metadata merge test (#4119)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 319ec30b7f Adds compaction metadata merge test (#4119)
319ec30b7f is described below
commit 319ec30b7f28c5b740b02aab3b935da465956f22
Author: Daniel Roberts
AuthorDate: Wed Jan 3 20:31:10 2024 -0500
Adds compaction metadata merge test (#4119)
---
.../apache/accumulo/test/functional/MergeIT.java | 63 ++
1 file changed, 63 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index 82a7264207..43590e8030 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@ -20,14 +20,19 @@ package org.apache.accumulo.test.functional;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.UUID;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@ -37,10 +42,20 @@ import
org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
+import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.util.Merge;
+import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
@@ -215,4 +230,52 @@ public class MergeIT extends AccumuloClusterHarness {
}
}
}
+
+ // Test that merge handles metadata from compactions
+ @Test
+ public void testCompactionMetadata() throws Exception {
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ var split = new Text("m");
+ c.tableOperations().addSplits(tableName, new TreeSet<>(List.of(split)));
+
+ TableId tableId = getServerContext().getTableId(tableName);
+
+ // add metadata from compactions to tablets prior to merge
+ try (var tabletsMutator = getServerContext().getAmple().mutateTablets())
{
+for (var extent : List.of(new KeyExtent(tableId, split, null),
+new KeyExtent(tableId, null, split))) {
+ var tablet = tabletsMutator.mutateTablet(extent);
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+
+ TabletFile tmpFile = new TabletFile(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
+ CompactionExecutorId ceid =
CompactionExecutorIdImpl.externalId("G1");
+ Set jobFiles =
+ Set.of(new
StoredTabletFile("file:///accumulo/tables/t-0/b-0/b2.rf"));
+ ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, jobFiles,
+ tmpFile, "localhost:", CompactionKind.SYSTEM, (short) 2,
ceid, false, false, 44L);
+ tablet.putExternalCompaction(ecid, ecMeta);
+ tablet.mutate();
+}
+ }
+
+ // ensure data is in metadata table as expected
+ try (var tablets =
getServerContext().getAmple().readTablets().forTable(tableId).build()) {
+for (var tablet : tablets) {
+ assertFalse(tablet.getExternalCompactions().isEmpty());
+}
+ }
+
+ c.tableOperations().merge(tableName, null, null);
+
+ // ensure merge operation remove compaction entries
+ try (var tablets =
getServerContext().getAmple().readTablets().forTable(tableId).build()) {
+for (var tablet : tablets) {
+ assertTrue(tablet.getExternalCompactions().isEmpty());
+}
+ }
+}
+ }
}
(accumulo-testing) branch main updated: Fixes property deprecation build errors
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 725b723 Fixes property deprecation build errors
new 6611044 Merge pull request #269 from
ddanielr/fix-property-deprecations
725b723 is described below
commit 725b723cb2c1e9310fc67298526a47d30a743a80
Author: Daniel Roberts
AuthorDate: Tue Jan 2 17:42:55 2024 +
Fixes property deprecation build errors
---
conf/accumulo-testing.properties| 4 ++--
.../accumulo/testing/randomwalk/concurrent/Config.java | 13 +++--
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/conf/accumulo-testing.properties b/conf/accumulo-testing.properties
index 85b3349..b2cbb5e 100644
--- a/conf/accumulo-testing.properties
+++ b/conf/accumulo-testing.properties
@@ -45,8 +45,8 @@ test.ci.common.accumulo.num.tablets=20
test.ci.common.auths=
# Accumulo tserver properties to set when creating a table
test.ci.common.accumulo.server.props=\
-tserver.compaction.major.service.cs1.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
\
-tserver.compaction.major.service.cs1.planner.opts.executors=\
+compaction.service.cs1.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
\
+compaction.service.cs1.planner.opts.executors=\
[{"name":"small","type":"internal","maxSize":"16M","numThreads":8},\
{"name":"medium","type":"internal","maxSize":"128M","numThreads":4},\
{"name":"large","type":"internal","numThreads":2}]
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
index b6e3a2f..ccd3d81 100644
---
a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
+++
b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Config.java
@@ -56,15 +56,24 @@ public class Config extends Test {
final Property TSERV_READ_AHEAD_MAXCONCURRENT_deprecated =
Property.TSERV_SCAN_EXECUTORS_DEFAULT_THREADS;
+
+ @SuppressWarnings("deprecation")
+ final Property TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN_deprecated =
+ Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN;
+
+ @SuppressWarnings("deprecation")
+ final Property TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS_deprecated =
+ Property.TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS;
+
// @formatter:off
final Setting[] settings = {
s(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, 1, 10),
s(Property.TSERV_DATACACHE_SIZE, 0, 10L),
s(Property.TSERV_INDEXCACHE_SIZE, 0, 10L),
s(Property.TSERV_CLIENT_TIMEOUT, 100, 1),
- s(Property.TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS,
1, 10),
+
s(TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS_deprecated, 1, 10),
s(Property.TSERV_MAJC_DELAY, 100, 1),
- s(Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN,
3, 100),
+ s(TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN_deprecated,
3, 100),
s(Property.TSERV_MINC_MAXCONCURRENT, 1, 10),
s(Property.TSERV_DEFAULT_BLOCKSIZE, 10, 1000L),
s(Property.TSERV_MAX_IDLE, 1, 500 * 1000),
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 157695162242cf2ba247dda22e2d4ef871e4f978 Merge: 1bf7dcb184 3792630c86 Author: Daniel Roberts AuthorDate: Thu Dec 28 04:11:18 2023 + Merge branch 'main' into elasticity .../accumulo/server/metadata/ServerAmpleImpl.java | 18 ++ .../accumulo/test/functional/GarbageCollectorIT.java | 18 +++--- 2 files changed, 29 insertions(+), 7 deletions(-)
(accumulo) branch elasticity updated (1bf7dcb184 -> 1576951622)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1bf7dcb184 Merge branch 'main' into elasticity add d973179452 Remove mutation timestamp for valid GcCandidates (#4113) add 3792630c86 Merge branch '2.1' new 1576951622 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/server/metadata/ServerAmpleImpl.java | 18 ++ .../accumulo/test/functional/GarbageCollectorIT.java | 18 +++--- 2 files changed, 29 insertions(+), 7 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3792630c867b2affb4f16efc9f537a04ca66cf3e
Merge: b8869fb72e d973179452
Author: Daniel Roberts
AuthorDate: Thu Dec 28 04:08:36 2023 +
Merge branch '2.1'
.../accumulo/server/metadata/ServerAmpleImpl.java | 18 ++
.../accumulo/test/functional/GarbageCollectorIT.java | 18 +++---
2 files changed, 29 insertions(+), 7 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index 0e4fa2c478,ef02f22b4d..7aa6ecb6e1
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@@ -227,13 -226,23 +227,23 @@@ public class ServerAmpleImpl extends Am
}
try (BatchWriter writer = context.createBatchWriter(level.metaTable())) {
- for (GcCandidate candidate : candidates) {
- Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
- m.putDelete(EMPTY_TEXT, EMPTY_TEXT, candidate.getUid());
- writer.addMutation(m);
+ if (type == GcCandidateType.VALID) {
+ for (GcCandidate candidate : candidates) {
+ Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
+ // Removes all versions of the candidate to avoid reprocessing
deleted file entries
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+ writer.addMutation(m);
+ }
+ } else {
+ for (GcCandidate candidate : candidates) {
+ Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
+ // Removes this and older versions while allowing newer candidate
versions to persist
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT, candidate.getUid());
+ writer.addMutation(m);
+ }
}
} catch (MutationsRejectedException | TableNotFoundException e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
}
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 82c657e1a5,1776bc4c03..01ca7b8f5d
---
a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@@ -497,11 -510,10 +506,11 @@@ public class GarbageCollectorIT extend
GcCandidate deleteCandidate = candidates.get(0);
assertNotNull(deleteCandidate);
-ample.putGcCandidates(tableId, List.of(new
StoredTabletFile(deleteCandidate.getPath(;
+ample.putGcCandidates(tableId,
+List.of(StoredTabletFile.of(new Path(deleteCandidate.getPath();
log.debug("Deleting Candidate {}", deleteCandidate);
- ample.deleteGcCandidates(datalevel, List.of(deleteCandidate),
Ample.GcCandidateType.INUSE);
+ ample.deleteGcCandidates(datalevel, List.of(deleteCandidate), type);
candidate = ample.getGcCandidates(datalevel);
(accumulo) branch main updated (b8869fb72e -> 3792630c86)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b8869fb72e Merge branch '2.1' add d973179452 Remove mutation timestamp for valid GcCandidates (#4113) new 3792630c86 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/server/metadata/ServerAmpleImpl.java | 18 ++ .../accumulo/test/functional/GarbageCollectorIT.java | 18 +++--- 2 files changed, 29 insertions(+), 7 deletions(-)
(accumulo) branch 2.1 updated: Remove mutation timestamp for valid GcCandidates (#4113)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d973179452 Remove mutation timestamp for valid GcCandidates (#4113)
d973179452 is described below
commit d97317945290fcd4e3a4eb8b3cf7e427fcc322bd
Author: Daniel Roberts
AuthorDate: Wed Dec 27 23:07:39 2023 -0500
Remove mutation timestamp for valid GcCandidates (#4113)
* Remove mutation timestamp for valid GcCandidates
---
.../accumulo/server/metadata/ServerAmpleImpl.java | 18 ++
.../accumulo/test/functional/GarbageCollectorIT.java | 18 +++---
2 files changed, 29 insertions(+), 7 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index 223a9cf112..ef02f22b4d 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -226,10 +226,20 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
}
try (BatchWriter writer = context.createBatchWriter(level.metaTable())) {
- for (GcCandidate candidate : candidates) {
-Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
-m.putDelete(EMPTY_TEXT, EMPTY_TEXT, candidate.getUid());
-writer.addMutation(m);
+ if (type == GcCandidateType.VALID) {
+for (GcCandidate candidate : candidates) {
+ Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
+ // Removes all versions of the candidate to avoid reprocessing
deleted file entries
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+ writer.addMutation(m);
+}
+ } else {
+for (GcCandidate candidate : candidates) {
+ Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
+ // Removes this and older versions while allowing newer candidate
versions to persist
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT, candidate.getUid());
+ writer.addMutation(m);
+}
}
} catch (MutationsRejectedException | TableNotFoundException e) {
throw new RuntimeException(e);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index d7899e1a3b..1776bc4c03 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -285,6 +285,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase
{
log.info("User GcCandidate Deletion test of table: {}", table);
log.info("GcCandidates will be added/removed from table: {}",
DataLevel.USER.metaTable());
createAndDeleteUniqueMutation(TableId.of(table),
Ample.GcCandidateType.INUSE);
+ createAndDeleteUniqueMutation(TableId.of(table),
Ample.GcCandidateType.VALID);
}
}
@@ -295,6 +296,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase
{
log.info("Metadata GcCandidate Deletion test");
log.info("GcCandidates will be added/removed from table: {}",
DataLevel.METADATA.metaTable());
createAndDeleteUniqueMutation(tableId, Ample.GcCandidateType.INUSE);
+createAndDeleteUniqueMutation(tableId, Ample.GcCandidateType.VALID);
}
/**
@@ -467,6 +469,13 @@ public class GarbageCollectorIT extends
ConfigurableMacBase {
}
private void createAndDeleteUniqueMutation(TableId tableId,
Ample.GcCandidateType type) {
+int totalCandidates = 2;
+boolean inUseDelete = true;
+
+if (type == Ample.GcCandidateType.VALID) {
+ totalCandidates = 1;
+ inUseDelete = false;
+}
Ample ample = cluster.getServerContext().getAmple();
DataLevel datalevel = Ample.DataLevel.of(tableId);
@@ -504,7 +513,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase
{
ample.putGcCandidates(tableId, List.of(new
StoredTabletFile(deleteCandidate.getPath(;
log.debug("Deleting Candidate {}", deleteCandidate);
-ample.deleteGcCandidates(datalevel, List.of(deleteCandidate),
Ample.GcCandidateType.INUSE);
+ample.deleteGcCandidates(datalevel, List.of(deleteCandidate), type);
candidate = ample.getGcCandidates(datalevel);
@@ -519,7 +528,10 @@ public class GarbageCollectorIT extends
ConfigurableMacBase {
}
counter++;
}
-assertEquals(2, counter);
-assertTrue(foundNewCandidate);
+assertEquals(totalCandidates, counter);
+assertEquals(inUseDelete, foundNewCandidate);
+
+// Cleanup from test
+
(accumulo) branch elasticity updated (8960452814 -> 1bf7dcb184)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8960452814 Merge branch 'main' into elasticity add fdd73fb8b2 Fix bug in compaction props (#4117) add b8869fb72e Merge branch '2.1' new 1bf7dcb184 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../spi/compaction/DefaultCompactionPlanner.java | 6 +- .../compaction/CompactionPlannerInitParams.java| 2 +- .../compaction/DefaultCompactionPlannerTest.java | 365 +++-- 3 files changed, 120 insertions(+), 253 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 1bf7dcb184cb2c2ed9e9a4768370a7980a6c22bd
Merge: 8960452814 b8869fb72e
Author: Daniel Roberts
AuthorDate: Wed Dec 27 23:37:54 2023 +
Merge branch 'main' into elasticity
.../spi/compaction/DefaultCompactionPlanner.java | 6 +-
.../compaction/CompactionPlannerInitParams.java| 2 +-
.../compaction/DefaultCompactionPlannerTest.java | 365 +++--
3 files changed, 120 insertions(+), 253 deletions(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
index e25b7c519b,bafda93e3e..6447a2d147
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
@@@ -29,15 -27,16 +28,18 @@@ import static org.junit.jupiter.api.Ass
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collection;
+import java.util.Collections;
+ import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+ import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
+ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
@@@ -243,152 -268,9 +271,167 @@@ public class DefaultCompactionPlannerTe
plan = planner.makePlan(params);
job = getOnlyElement(plan.getJobs());
assertEquals(all, job.getFiles());
- assertEquals(CompactionExecutorIdImpl.externalId("large"),
job.getExecutor());
+ assertEquals(CompactionExecutorIdImpl.internalId(csid, "large"),
job.getExecutor());
}
+ @Test
+ public void testMultipleCompactions() {
+// This test validates that when a tablet has many files that multiple
compaction jobs can be
+// issued at the same time.
++String executors = "[{'name':'small','type':
'internal','maxSize':'32M','numThreads':1},"
+++ "{'name':'medium','type':
'internal','maxSize':'128M','numThreads':2},"
+++ "{'name':'large','type':
'internal','maxSize':'512M','numThreads':3}]";
++
+for (var kind : List.of(CompactionKind.USER, CompactionKind.SYSTEM)) {
- var planner = createPlanner(false);
++ var planner = createPlanner(defaultConf, executors);
+ var all = IntStream.range(0, 990).mapToObj(i -> createCF("F" + i,
1000)).collect(toSet());
+ // simulate 10 larger files, these should not compact at the same time
as the smaller files.
+ // Its more optimal to wait for all of the smaller files to compact and
them compact the
+ // output of compacting the smaller files with the larger files.
+ IntStream.range(990, 1000).mapToObj(i -> createCF("C" + i,
2)).forEach(all::add);
+ var params = createPlanningParams(all, all, Set.of(), 2, kind);
+ var plan = planner.makePlan(params);
+
+ // There are 990 smaller files to compact. Should produce 66 jobs of 15
smaller files each.
+ assertEquals(66, plan.getJobs().size());
+ Set filesSeen = new HashSet<>();
+ plan.getJobs().forEach(job -> {
+assertEquals(15, job.getFiles().size());
+assertEquals(kind, job.getKind());
- assertEquals(CompactionExecutorIdImpl.externalId("small"),
job.getExecutor());
++assertEquals(CompactionExecutorIdImpl.internalId(csid, "small"),
job.getExecutor());
+// ensure the files across all of the jobs are disjoint
+job.getFiles().forEach(cf -> assertTrue(filesSeen.add(cf)));
+ });
+
+ // Ensure all of the smaller files are scheduled for compaction. Should
not see any of the
+ // larger files.
+ assertEquals(IntStream.range(0, 990).mapToObj(i -> createCF("F" + i,
1000)).collect(toSet()),
+ filesSeen);
+}
+ }
+
+ @Test
+ public void testMultipleCompactionsAndLargeCompactionRatio() {
- var planner = createPlanner(false);
++
++String executors = "[{'name':'small','type':
'internal','maxSize':'32M','numThreads':1},"
+++ "{'name':'medium','type':
'internal','maxSize':'128M','numThreads':2},"
+++ "{'name':'large','type':
'internal','maxSize':'512M','numThreads':3}]";
++var planner = createPlanner(defaultConf, executors);
+var all = IntStream.range(0, 65).mapToObj(i -> createC
(accumulo) branch main updated (829634653b -> b8869fb72e)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 829634653b Merge branch '2.1' add fdd73fb8b2 Fix bug in compaction props (#4117) new b8869fb72e Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../spi/compaction/DefaultCompactionPlanner.java | 6 +- .../compaction/CompactionPlannerInitParams.java| 2 +- .../compaction/DefaultCompactionPlannerTest.java | 343 ++--- 3 files changed, 106 insertions(+), 245 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b8869fb72e2aa58736250c52f3366f196ac86c79
Merge: 829634653b fdd73fb8b2
Author: Daniel Roberts
AuthorDate: Wed Dec 27 20:30:47 2023 +
Merge branch '2.1'
.../spi/compaction/DefaultCompactionPlanner.java | 6 +-
.../compaction/CompactionPlannerInitParams.java| 2 +-
.../compaction/DefaultCompactionPlannerTest.java | 343 ++---
3 files changed, 106 insertions(+), 245 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index b73544143e,9385806831..8f7969519a
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@@ -279,106 -210,105 +280,107 @@@ public class DefaultCompactionPlanner i
determineMaxFilesToCompact(params);
}
- @SuppressWarnings("removal")
++ @SuppressWarnings("deprecation")
private void determineMaxFilesToCompact(InitParameters params) {
-String fqo = params.getFullyQualifiedOption("maxOpen");
-if (!params.getServiceEnvironment().getConfiguration().isSet(fqo)
-&& params.getServiceEnvironment().getConfiguration()
-.isSet(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey())) {
- log.warn("The property " + Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()
- + " was set, it is deprecated. Set the " + fqo + " option
instead.");
- this.maxFilesToCompact =
Integer.parseInt(params.getServiceEnvironment().getConfiguration()
- .get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()));
-} else {
- this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen",
-
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue()));
+
+String maxOpen = params.getOptions().get("maxOpen");
+if (maxOpen == null) {
- maxOpen = "10";
- log.trace("default maxOpen not set, defaulting to 10");
++ maxOpen =
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue();
++ log.trace("default maxOpen not set, defaulting to {}", maxOpen);
}
+this.maxFilesToCompact = Integer.parseInt(maxOpen);
}
- @Override
- public CompactionPlan makePlan(PlanningParameters params) {
-try {
+ private void validateConfig(JsonElement json, List fields, String
className) {
- if (params.getCandidates().isEmpty()) {
-return params.createPlanBuilder().build();
- }
+JsonObject jsonObject = GSON.get().fromJson(json, JsonObject.class);
+
+List objectProperties = new ArrayList<>(jsonObject.keySet());
+HashSet classFieldNames = new HashSet<>(fields);
+
+if (!classFieldNames.containsAll(objectProperties)) {
+ objectProperties.removeAll(classFieldNames);
+ throw new JsonParseException(
+ "Invalid fields: " + objectProperties + " provided for class: " +
className);
+}
+ }
- Set filesCopy = new HashSet<>(params.getCandidates());
+ @Override
+ public CompactionPlan makePlan(PlanningParameters params) {
+if (params.getCandidates().isEmpty()) {
+ return params.createPlanBuilder().build();
+}
- long maxSizeToCompact = getMaxSizeToCompact(params.getKind());
+Set filesCopy = new HashSet<>(params.getCandidates());
- Collection group;
- if (params.getRunningCompactions().isEmpty()) {
-group = findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
-maxSizeToCompact);
+long maxSizeToCompact = getMaxSizeToCompact(params.getKind());
-if (!group.isEmpty() && group.size() < params.getCandidates().size()
-&& params.getCandidates().size() <= maxFilesToCompact
-&& (params.getKind() == CompactionKind.USER
-|| params.getKind() == CompactionKind.SELECTOR)) {
- // USER and SELECTOR compactions must eventually compact all files.
When a subset of files
- // that meets the compaction ratio is selected, look ahead and see
if the next compaction
- // would also meet the compaction ratio. If not then compact
everything to avoid doing
- // more than logarithmic work across multiple comapctions.
+Collection group;
+if (params.getRunningCompactions().isEmpty()) {
+ group =
+ findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact, maxSizeToCompact);
- filesCopy.removeAll(group);
- filesCopy.add(getExpected(group, 0));
+ if (!group.isEmpty() &&
(accumulo) branch 2.1 updated: Fix bug in compaction props (#4117)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new fdd73fb8b2 Fix bug in compaction props (#4117)
fdd73fb8b2 is described below
commit fdd73fb8b27c5e49f91f0c1008457658a588a411
Author: Christopher Tubbs
AuthorDate: Wed Dec 27 14:38:10 2023 -0500
Fix bug in compaction props (#4117)
This fixes a bug in the compaction properties to ensure the replacement
property is always preferred over the deprecated open.max compaction
property when it is set.
Add tests for maxOpen to override open.max:
* Adds a test to ensure that setting the maxOpen option for a compaction
service will override the deprecated `open.max` property if set
* Condenses helper methods in planner tests
* Uses CompactionPlannerInitParams for tests instead of custom test code
* Adds test case for default compaction service used with deprecated
property
* Removes hardcoded maxOpen value with reference to default property
value
* Modifies the getFullyQualifiedOption to return the correct path for
the `.planner.opts.` properties
This is a reapplication of #4092 after it was reverted, to use
SiteConfiguration for testing overrides rather than modifications to
ConfigurationCopy
Changes made by ctubbsii that diverge from #4092:
* Update commit log message to add detail and format it
* Omit changes to ConfigurationCopy, including changes in #4112, which
is now OBE, to add a parent to preserve its role as a simple "flat"
configuration object for testing and simple operations
* Use SiteConfiguration with overrides, instead of ConfigurationCopy
with a parent, to test override behavior for
DefaultCompactionPlannerTest
Co-authored-by: Daniel Roberts
---
.../spi/compaction/DefaultCompactionPlanner.java | 3 +-
.../compaction/CompactionPlannerInitParams.java| 2 +-
.../compaction/DefaultCompactionPlannerTest.java | 374 +++--
3 files changed, 204 insertions(+), 175 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index be8e25299b..9385806831 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -221,7 +221,8 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
this.maxFilesToCompact =
Integer.parseInt(params.getServiceEnvironment().getConfiguration()
.get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()));
} else {
- this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen", "10"));
+ this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen",
+
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue()));
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
index 0f79ce4df0..eb287153d6 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
@@ -60,7 +60,7 @@ public class CompactionPlannerInitParams implements
CompactionPlanner.InitParame
@Override
public String getFullyQualifiedOption(String key) {
-return Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey() + serviceId +
".opts." + key;
+return Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey() + serviceId +
".planner.opts." + key;
}
@Override
diff --git
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
index 568b57cd2d..ab2003841b 100644
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
@@ -22,27 +22,35 @@ import static
com.google.common.collect.MoreCollectors.onlyElement;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collection;
+import java.util.HashMa
(accumulo) branch 2.1 updated: Update isPropertySet to check parent config (#4112)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 4c3085b1f3 Update isPropertySet to check parent config (#4112)
4c3085b1f3 is described below
commit 4c3085b1f3045698fd37c620fe0ec263757fbfb9
Author: Daniel Roberts
AuthorDate: Fri Dec 22 13:48:39 2023 -0500
Update isPropertySet to check parent config (#4112)
---
.../main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
index e487e2840a..60502608fa 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
@@ -138,6 +138,10 @@ public class ConfigurationCopy extends
AccumuloConfiguration {
@Override
public boolean isPropertySet(Property prop) {
-return copy.containsKey(prop.getKey());
+if (parent != null) {
+ return copy.containsKey(prop.getKey()) || parent.isPropertySet(prop);
+} else {
+ return copy.containsKey(prop.getKey());
+}
}
}
(accumulo) branch 2.1 updated: Fix bug in compaction props (#4092)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 67fd967415 Fix bug in compaction props (#4092)
67fd967415 is described below
commit 67fd967415612ab0770b422a5b24430655275946
Author: Daniel Roberts
AuthorDate: Fri Dec 22 00:16:17 2023 -0500
Fix bug in compaction props (#4092)
* Add tests for maxOpen to override open.max
Adds a test to ensure that setting the maxOpen option for a compaction
service will override the deprecated `open.max` property if set.
Condenses helper methods in planner tests
Uses CompactionPlannerInitParams for tests instead of custom test code
Modifies the ConfigurationCopy class to accept a parent conf to allow
for testing user set properties vs default values in test cases
Adds test case for default compaction service used with deprecated
property
Removes hardcoded maxOpen value with reference to default property value
Modifies the getFullyQualifiedOption to return the correct path for the
`.planner.opts.` properties.
---
.../accumulo/core/conf/ConfigurationCopy.java | 24 +-
.../spi/compaction/DefaultCompactionPlanner.java | 3 +-
.../compaction/CompactionPlannerInitParams.java| 2 +-
.../compaction/DefaultCompactionPlannerTest.java | 361 +++--
4 files changed, 214 insertions(+), 176 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
index 14424c196a..e487e2840a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationCopy.java
@@ -30,6 +30,8 @@ import java.util.stream.Stream;
* configuration
*/
public class ConfigurationCopy extends AccumuloConfiguration {
+
+ AccumuloConfiguration parent = null;
private long updateCount = 0;
final Map copy = Collections.synchronizedMap(new HashMap<>());
@@ -42,6 +44,17 @@ public class ConfigurationCopy extends AccumuloConfiguration
{
this(config.entrySet());
}
+ /**
+ * Creates a new configuration
+ *
+ * @param config - configuration property key/value pairs to copy
+ * @param parent - Higher level accumulo config to allow for property
overrides
+ */
+ public ConfigurationCopy(Map config, AccumuloConfiguration
parent) {
+this(config.entrySet());
+this.parent = parent;
+ }
+
/**
* Creates a new configuration.
*
@@ -69,11 +82,20 @@ public class ConfigurationCopy extends
AccumuloConfiguration {
@Override
public String get(Property property) {
-return copy.get(property.getKey());
+if (copy.containsKey(property.getKey())) {
+ return copy.get(property.getKey());
+} else if (parent != null) {
+ return parent.get(property);
+} else {
+ return null;
+}
}
@Override
public void getProperties(Map props, Predicate
filter) {
+if (parent != null) {
+ parent.getProperties(props, filter);
+}
for (Entry entry : copy.entrySet()) {
if (filter.test(entry.getKey())) {
props.put(entry.getKey(), entry.getValue());
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index be8e25299b..9385806831 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -221,7 +221,8 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
this.maxFilesToCompact =
Integer.parseInt(params.getServiceEnvironment().getConfiguration()
.get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()));
} else {
- this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen", "10"));
+ this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen",
+
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue()));
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
index 0f79ce4df0..eb287153d6 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
@@ -60,7 +60,7 @@ public class CompactionPlannerInitParams implements
CompactionPlanner.InitParame
@Override
public Strin
(accumulo) branch elasticity updated: Enables CompactionExecutorIT tests (#4080)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5ed277 Enables CompactionExecutorIT tests (#4080)
5ed277 is described below
commit 5ed277a3ad2773551860d7f38b3470bbefb1
Author: Daniel Roberts
AuthorDate: Mon Dec 18 22:26:42 2023 -0500
Enables CompactionExecutorIT tests (#4080)
* Adds additional method to stop compactors by resource group names.
* Remove failing compaction selector test
-
Co-authored-by: Keith Turner
---
.../MiniAccumuloClusterControl.java| 20 +++
.../test/compaction/CompactionExecutorIT.java | 165 ++---
2 files changed, 134 insertions(+), 51 deletions(-)
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index 6d3b36026e..b3977dab00 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@ -254,6 +254,26 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
stop(server, null);
}
+ public void stopCompactorGroup(String compactorResourceGroup) {
+synchronized (compactorProcesses) {
+ var group = compactorProcesses.get(compactorResourceGroup);
+ if (group == null) {
+return;
+ }
+ group.forEach(process -> {
+try {
+ cluster.stopProcessWithTimeout(process, 30, TimeUnit.SECONDS);
+} catch (ExecutionException | TimeoutException e) {
+ log.warn("Compactor did not fully stop after 30 seconds", e);
+ throw new RuntimeException(e);
+} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+}
+ });
+ compactorProcesses.remove(compactorResourceGroup);
+}
+ }
+
@Override
public synchronized void stop(ServerType server, String hostname) throws
IOException {
switch (server) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
index a7309cb693..19f6d0bb30 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionExecutorIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.test.compaction;
+import static org.apache.accumulo.core.util.LazySingletons.GSON;
import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -29,7 +30,7 @@ import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
-import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -38,6 +39,7 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@ -67,36 +69,50 @@ import
org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.spi.compaction.CompactionPlan;
import org.apache.accumulo.core.spi.compaction.CompactionPlanner;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
-@Disabled // ELASTICITY_TODO
public class CompactionExecutorIT extends SharedMiniClusterBase {
+ public static final List compactionGroups = new LinkedList<>();
+ public static final Logger log =
LoggerFactory.getLogger(CompactionExecutorIT.class);
public static class TestPlanner implements CompactionPlanner {
+private static class ExecutorConfig {
(accumulo) branch elasticity updated: Handle exception when compaction queue is empty (#4079)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 2e691d5cfa Handle exception when compaction queue is empty (#4079)
2e691d5cfa is described below
commit 2e691d5cfaaa98a4536f75d45a461db8ff24a685
Author: Daniel Roberts
AuthorDate: Fri Dec 15 10:10:43 2023 -0500
Handle exception when compaction queue is empty (#4079)
* Handle exception if compaction queue is empty
Handles a NullElementException if `getLowestPriority` is called on an
empty compaction priority queue.
* Refactor getLowestPriority to avoid exception
---
.../accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
index ae1c278147..74f168a985 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
@@ -158,6 +158,9 @@ public class CompactionJobPriorityQueue {
}
public synchronized long getLowestPriority() {
+if (jobQueue.isEmpty()) {
+ return 0;
+}
return jobQueue.lastKey().job.getPriority();
}
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 593711095aba3847aa5e1b773a5cdcde0c4bacfc Merge: 4d38127e5a 5b46bd8f97 Author: Daniel Roberts AuthorDate: Wed Dec 13 19:58:32 2023 + Merge branch '2.1' .../org/apache/accumulo/test/fate/zookeeper/FateIT.java | 13 ++--- 1 file changed, 10 insertions(+), 3 deletions(-)
(accumulo) branch main updated (4d38127e5a -> 593711095a)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4d38127e5a Merge branch '2.1' add 5b46bd8f97 Fixes Fate IT race conditions (#4029) new 593711095a Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/test/fate/zookeeper/FateIT.java | 13 ++--- 1 file changed, 10 insertions(+), 3 deletions(-)
(accumulo) branch elasticity updated (4fef06761c -> 09e1c00983)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4fef06761c Merge branch 'main' into elasticity add e62ff01808 Make bulk import wait for in-progress bulk imports (#4059) add 4d38127e5a Merge branch '2.1' new 09e1c00983 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 09e1c009830e47f009b438b187794113e65781cb Merge: 4fef06761c 4d38127e5a Author: Daniel Roberts AuthorDate: Wed Dec 13 18:47:21 2023 + Merge branch 'main' into elasticity
(accumulo) branch main updated (1aa81d81b0 -> 4d38127e5a)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1aa81d81b0 Merge branch '2.1' add e62ff01808 Make bulk import wait for in-progress bulk imports (#4059) new 4d38127e5a Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/tablet/Tablet.java | 45 ++ 1 file changed, 30 insertions(+), 15 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4d38127e5a3f10e404e90323c49cf146f4c70007
Merge: 1aa81d81b0 e62ff01808
Author: Daniel Roberts
AuthorDate: Wed Dec 13 18:34:15 2023 +
Merge branch '2.1'
.../org/apache/accumulo/tserver/tablet/Tablet.java | 45 ++
1 file changed, 30 insertions(+), 15 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 7c741729df,6a646c4f57..65db951634
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1733,17 -1750,40 +1733,40 @@@ public class Tablet extends TabletBase
throw new IOException("tablet " + extent + " is closed");
}
- // TODO check seems unneeded now - ACCUMULO-1291
- long lockWait = System.currentTimeMillis() - now;
- if (lockWait
- >
getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT))
{
- throw new IOException(
- "Timeout waiting " + (lockWait / 1000.) + " seconds to get tablet
lock for " + extent);
+ long rpcTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(
+ (long)
(getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)
+ * 1.1));
+
+ // wait for any files that are bulk importing up to the RPC timeout
limit
+ while (!Collections.disjoint(bulkImporting, fileMap.keySet())) {
+ try {
+ wait(1_000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+
+ long lockWait = System.nanoTime() - now;
+ if (lockWait > rpcTimeoutNanos) {
+ throw new IOException("Timeout waiting " +
TimeUnit.NANOSECONDS.toSeconds(lockWait)
+ + " seconds to get tablet lock for " + extent + " " + tid);
+ }
+ }
+
+ // need to check this again because when wait is called above the lock
is released.
+ if (isClosed()) {
+ throw new IOException("tablet " + extent + " is closed");
+ }
+
+ long lockWait = System.nanoTime() - now;
+ if (lockWait > rpcTimeoutNanos) {
+ throw new IOException("Timeout waiting " +
TimeUnit.NANOSECONDS.toSeconds(lockWait)
+ + " seconds to get tablet lock for " + extent + " " + tid);
}
- List alreadyImported = bulkImported.get(tid);
+ List alreadyImported = bulkImported.get(tid);
if (alreadyImported != null) {
-for (TabletFile entry : alreadyImported) {
+for (ReferencedTabletFile entry : alreadyImported) {
if (fileMap.remove(entry) != null) {
log.trace("Ignoring import of bulk file already imported: {}",
entry);
}
(accumulo) branch 2.1 updated: Make bulk import wait for in-progress bulk imports (#4059)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e62ff01808 Make bulk import wait for in-progress bulk imports (#4059)
e62ff01808 is described below
commit e62ff01808c1641ba6d67ca526d834db9dae38da
Author: Daniel Roberts
AuthorDate: Wed Dec 13 13:20:06 2023 -0500
Make bulk import wait for in-progress bulk imports (#4059)
* Make bulk import wait for in-progress bulk imports
-
Co-authored-by: Keith Turner
---
.../org/apache/accumulo/tserver/tablet/Tablet.java | 45 ++
1 file changed, 30 insertions(+), 15 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 3f7eebe95f..6a646c4f57 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1744,18 +1744,41 @@ public class Tablet extends TabletBase {
// Clients timeout and will think that this operation failed.
// Don't do it if we spent too long waiting for the lock
-long now = System.currentTimeMillis();
+long now = System.nanoTime();
synchronized (this) {
if (isClosed()) {
throw new IOException("tablet " + extent + " is closed");
}
- // TODO check seems unneeded now - ACCUMULO-1291
- long lockWait = System.currentTimeMillis() - now;
- if (lockWait
- >
getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT))
{
-throw new IOException(
-"Timeout waiting " + (lockWait / 1000.) + " seconds to get tablet
lock for " + extent);
+ long rpcTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(
+ (long)
(getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)
+ * 1.1));
+
+ // wait for any files that are bulk importing up to the RPC timeout limit
+ while (!Collections.disjoint(bulkImporting, fileMap.keySet())) {
+try {
+ wait(1_000);
+} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+}
+
+long lockWait = System.nanoTime() - now;
+if (lockWait > rpcTimeoutNanos) {
+ throw new IOException("Timeout waiting " +
TimeUnit.NANOSECONDS.toSeconds(lockWait)
+ + " seconds to get tablet lock for " + extent + " " + tid);
+}
+ }
+
+ // need to check this again because when wait is called above the lock
is released.
+ if (isClosed()) {
+throw new IOException("tablet " + extent + " is closed");
+ }
+
+ long lockWait = System.nanoTime() - now;
+ if (lockWait > rpcTimeoutNanos) {
+throw new IOException("Timeout waiting " +
TimeUnit.NANOSECONDS.toSeconds(lockWait)
++ " seconds to get tablet lock for " + extent + " " + tid);
}
List alreadyImported = bulkImported.get(tid);
@@ -1767,14 +1790,6 @@ public class Tablet extends TabletBase {
}
}
- fileMap.keySet().removeIf(file -> {
-if (bulkImporting.contains(file)) {
- log.info("Ignoring import of bulk file currently importing: " +
file);
- return true;
-}
-return false;
- });
-
if (fileMap.isEmpty()) {
return;
}
(accumulo) branch elasticity updated (e6799c9cf7 -> 4fef06761c)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from e6799c9cf7 Fix CompactionConfigChangeIT (#4060) add 6cfa1b2ea7 Remove redundant LogEntry value (#3997) add 01f48adf38 Remove active FateTx constraint for bulk import (#4063) add 1aa81d81b0 Merge branch '2.1' new 4fef06761c Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/MutationsRejectedException.java| 11 ++-- .../core/metadata/schema/TabletMutatorBase.java| 2 +- .../accumulo/core/tabletserver/log/LogEntry.java | 61 ++ .../core/metadata/schema/TabletMetadataTest.java | 11 ++-- .../core/tabletserver/log/LogEntryTest.java| 11 ++-- .../server/constraints/MetadataConstraints.java| 2 +- .../org/apache/accumulo/server/fs/VolumeUtil.java | 10 ++-- .../accumulo/server/util/ListVolumesUsed.java | 2 +- .../constraints/MetadataConstraintsTest.java | 56 +--- .../apache/accumulo/server/fs/VolumeUtilTest.java | 6 +-- .../server/manager/state/TabletManagementTest.java | 7 +-- .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 2 +- .../accumulo/manager/recovery/RecoveryManager.java | 2 +- .../org/apache/accumulo/tserver/TabletServer.java | 2 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +- .../test/MissingWalHeaderCompletesRecoveryIT.java | 6 +-- .../functional/TabletManagementIteratorIT.java | 4 +- 17 files changed, 77 insertions(+), 120 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4fef06761c2da98299414809f691ed85d937b4ed
Merge: e6799c9cf7 1aa81d81b0
Author: Daniel Roberts
AuthorDate: Wed Dec 13 17:58:00 2023 +
Merge branch 'main' into elasticity
.../core/client/MutationsRejectedException.java| 11 ++--
.../core/metadata/schema/TabletMutatorBase.java| 2 +-
.../accumulo/core/tabletserver/log/LogEntry.java | 61 ++
.../core/metadata/schema/TabletMetadataTest.java | 11 ++--
.../core/tabletserver/log/LogEntryTest.java| 11 ++--
.../server/constraints/MetadataConstraints.java| 2 +-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 10 ++--
.../accumulo/server/util/ListVolumesUsed.java | 2 +-
.../constraints/MetadataConstraintsTest.java | 56 +---
.../apache/accumulo/server/fs/VolumeUtilTest.java | 6 +--
.../server/manager/state/TabletManagementTest.java | 7 +--
.../accumulo/gc/GarbageCollectWriteAheadLogs.java | 2 +-
.../accumulo/manager/recovery/RecoveryManager.java | 2 +-
.../org/apache/accumulo/tserver/TabletServer.java | 2 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
.../test/MissingWalHeaderCompletesRecoveryIT.java | 6 +--
.../functional/TabletManagementIteratorIT.java | 4 +-
17 files changed, 77 insertions(+), 120 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
index ea13271f6f,4f7f9c230c..2792397ab7
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
@@@ -179,10 -173,10 +179,10 @@@ public abstract class TabletMutatorBase
}
@Override
- public Ample.TabletMutator putWal(LogEntry logEntry) {
+ public T putWal(LogEntry logEntry) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after
calling mutate.");
- mutation.put(LogColumnFamily.NAME, logEntry.getColumnQualifier(),
logEntry.getValue());
+ logEntry.addToMutation(mutation);
-return this;
+return getThis();
}
@Override
diff --cc
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 190f7e0e3b,617bf20e86..9fad119948
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -416,17 -374,9 +416,17 @@@ public class MetadataConstraints implem
case 7:
return "Lock not held in zookeeper by writer";
case 8:
- return "Bulk load transaction no longer running";
+ return "Bulk load mutation contains either inconsistent files or
multiple fateTX ids";
case 9:
+return "Malformed operation id";
+ case 10:
+return "Malformed hosting goal";
+ case 11:
+return "Malformed file selection value";
+ case 12:
return "Invalid data file metadata format";
+ case 13:
+return "Invalid compacted column";
}
return null;
}
diff --cc
server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 2bb653cb70,29f470ebe6..5c3ac18584
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@@ -130,59 -128,37 +130,59 @@@ public class VolumeUtil
}
}
- /**
- * This method does two things. First, it switches any volumes a tablet is
using that are
- * configured in instance.volumes.replacements. Second, if a tablet dir is
no longer configured
- * for use it chooses a new tablet directory.
- */
- public static TabletFiles updateTabletVolumes(ServerContext context,
ServiceLock zooLock,
- KeyExtent extent, TabletFiles tabletFiles) {
-List> replacements = context.getVolumeReplacements();
+ public static boolean needsVolumeReplacement(final List>
replacements,
+ final TabletMetadata tm) {
if (replacements.isEmpty()) {
- return tabletFiles;
+ return false;
}
-log.trace("Using volume replacements: {}", replacements);
-List logsToRemove = new ArrayList<>();
-List logsToAdd = new ArrayList<>();
+MutableBoolean needsReplacement = new MutableBoolean(false);
+
+Consumer consumer = le -> needsReplacement.setTrue();
+
+volumeReplacementEvaluation(replacements, tm, consumer, consumer,
+f -> needsReplacement.setTrue(), (f, dfv) ->
needsReplacement.setTrue());
+
+return needsReplacement.booleanValue();
+ }
-List filesToRemove = ne
(accumulo) branch main updated (6cfa1b2ea7 -> 1aa81d81b0)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 6cfa1b2ea7 Remove redundant LogEntry value (#3997) add 01f48adf38 Remove active FateTx constraint for bulk import (#4063) new 1aa81d81b0 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/MutationsRejectedException.java| 11 ++- .../server/constraints/MetadataConstraints.java| 20 + .../constraints/MetadataConstraintsTest.java | 94 -- 3 files changed, 24 insertions(+), 101 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 1aa81d81b0ff0da4dcd9ed4e33e599d701297c01
Merge: 6cfa1b2ea7 01f48adf38
Author: Daniel Roberts
AuthorDate: Wed Dec 13 15:35:48 2023 +
Merge branch '2.1'
.../core/client/MutationsRejectedException.java| 11 ++-
.../server/constraints/MetadataConstraints.java| 20 +
.../constraints/MetadataConstraintsTest.java | 94 --
3 files changed, 24 insertions(+), 101 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index aa9772a159,ced97d0a68..617bf20e86
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -390,9 -327,7 +374,9 @@@ public class MetadataConstraints implem
case 7:
return "Lock not held in zookeeper by writer";
case 8:
- return "Bulk load transaction no longer running";
+ return "Bulk load mutation contains either inconsistent files or
multiple fateTX ids";
+ case 9:
+return "Invalid data file metadata format";
}
return null;
}
diff --cc
server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
index 38f6d27ad2,bd59fef1e7..135a2bce60
---
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
@@@ -40,8 -33,6 +40,7 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.server.ServerContext;
- import org.apache.accumulo.server.zookeeper.TransactionWatcher.Arbitrator;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.easymock.EasyMock;
import org.junit.jupiter.api.Test;
@@@ -158,376 -128,77 +136,332 @@@ public class MetadataConstraintsTest
Mutation m;
List violations;
- // inactive txid
- m = new Mutation(new Text("0;foo"));
- m.put(
- BulkFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new Value("12345"));
- m.put(
- DataFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new DataFileValue(1, 1).encodeAsValue());
- assertViolation(mc, m, (short) 8);
-
- // txid that throws exception
- m = new Mutation(new Text("0;foo"));
- m.put(
- BulkFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new Value("9"));
- m.put(
- DataFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new DataFileValue(1, 1).encodeAsValue());
- assertViolation(mc, m, (short) 8);
-
- // active txid w/ file
+ // loaded marker w/ file
m = new Mutation(new Text("0;foo"));
-m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5"));
-m.put(DataFileColumnFamily.NAME, new Text("/someFile"),
+m.put(
+BulkFileColumnFamily.NAME, StoredTabletFile
+.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
+new Value("5"));
+m.put(
+DataFileColumnFamily.NAME, StoredTabletFile
+.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
new DataFileValue(1, 1).encodeAsValue());
violations = mc.check(createEnv(), m);
assertNull(violations);
- // active txid w/o file
+ // loaded marker w/o file
m = new Mutation(new Text("0;foo"));
-m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5"));
-violations = mc.check(createEnv(), m);
-assertNotNull(violations);
-assertEquals(1, violations.size());
-assertEquals(Short.valueOf((short) 8), violations.get(0));
+m.put(
+BulkFileColumnFamily.NAME, StoredTabletFile
+.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
+new Value("5"));
+assertViolation(mc, m, (short
(accumulo) branch 2.1 updated: Remove active FateTx constraint for bulk import (#4063)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 01f48adf38 Remove active FateTx constraint for bulk import (#4063)
01f48adf38 is described below
commit 01f48adf385faebb3bc2e5f3fb4e69dd4af12e3f
Author: Daniel Roberts
AuthorDate: Wed Dec 13 09:58:13 2023 -0500
Remove active FateTx constraint for bulk import (#4063)
* Remove active FateTx constraint for bulk import
Removes the active FateTx ID constraint for metadata mutations when
creating bulk import file markers.
Changes the MutationsRejectedException to return the violation type
codes.
-
Co-authored-by: Keith Turner
---
.../core/client/MutationsRejectedException.java| 18 +++--
.../server/constraints/MetadataConstraints.java| 22 +-
.../constraints/MetadataConstraintsTest.java | 84 --
3 files changed, 30 insertions(+), 94 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
b/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
index e3f773364c..30e6fbd183 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
import org.apache.accumulo.core.clientImpl.ClientContext;
@@ -59,8 +60,11 @@ public class MutationsRejectedException extends
AccumuloException {
Map> hashMap, Collection
serverSideErrors,
int unknownErrors, Throwable cause) {
super(
-"# constraint violations : " + cvsList.size() + " security codes: " +
hashMap.toString()
-+ " # server errors " + serverSideErrors.size() + " # exceptions
" + unknownErrors,
+"constraint violation codes : "
++
cvsList.stream().map(ConstraintViolationSummary::getViolationCode)
+.collect(Collectors.toSet())
++ " security codes: " + hashMap.toString() + " # server errors "
++ serverSideErrors.size() + " # exceptions " + unknownErrors,
cause);
this.cvsl.addAll(cvsList);
this.af.putAll(hashMap);
@@ -82,9 +86,13 @@ public class MutationsRejectedException extends
AccumuloException {
public MutationsRejectedException(AccumuloClient client,
List cvsList,
Map> hashMap, Collection
serverSideErrors,
int unknownErrors, Throwable cause) {
-super("# constraint violations : " + cvsList.size() + " security codes: "
-+ format(hashMap, (ClientContext) client) + " # server errors " +
serverSideErrors.size()
-+ " # exceptions " + unknownErrors, cause);
+super(
+"constraint violation codes : "
++
cvsList.stream().map(ConstraintViolationSummary::getViolationCode).collect(
+Collectors.toSet())
++ " security codes: " + format(hashMap, (ClientContext) client) +
" # server errors "
++ serverSideErrors.size() + " # exceptions " + unknownErrors,
+cause);
this.cvsl.addAll(cvsList);
this.af.putAll(hashMap);
this.es.addAll(serverSideErrors);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 1b5a074adb..ced97d0a68 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@ -24,10 +24,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
-import java.util.Objects;
import java.util.Set;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.ColumnUpdate;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -54,8 +52,6 @@ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Ta
import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.core.util.cleaner.CleanerUtil;
import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.zookeeper.TransactionWatcher.Arbitrator;
-import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
import org.apache.hadoop.io.Text;
import org.slf4j.Lo
(accumulo) branch elasticity updated: Fix CompactionConfigChangeIT (#4060)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e6799c9cf7 Fix CompactionConfigChangeIT (#4060)
e6799c9cf7 is described below
commit e6799c9cf7e3196e7af0aea3e679b07925503f2c
Author: Daniel Roberts
AuthorDate: Wed Dec 13 09:40:21 2023 -0500
Fix CompactionConfigChangeIT (#4060)
Modified the IT to use external compactors
---
.../test/compaction/CompactionConfigChangeIT.java | 35 ++
1 file changed, 22 insertions(+), 13 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
index 956d3bac67..22e8438695 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
@@ -33,14 +33,32 @@ import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.functional.SlowIterator;
import org.apache.accumulo.test.util.Wait;
-import org.junit.jupiter.api.Disabled;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.junit.jupiter.api.Test;
-@Disabled // ELASTICITY_TODO
public class CompactionConfigChangeIT extends AccumuloClusterHarness {
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
+
+cfg.getClusterServerConfiguration().addCompactorResourceGroup("e4", 1);
+cfg.getClusterServerConfiguration().addCompactorResourceGroup("e5", 1);
+
+cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner",
+DefaultCompactionPlanner.class.getName());
+cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner.opts.executors",
+("[{'name':'small','type':'external','maxSize':'2M', 'group': 'e1'},"
++ "{'name':'medium','type':'external','maxSize':'128M',
'group':'e2'},"
++
"{'name':'large','type':'external','group':'e3'}]").replaceAll("'", "\""));
+
+// use raw local file system so walogs sync and flush will work
+hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
public static long countFiles(AccumuloClient client, String table, String
fileNamePrefix)
throws Exception {
var ctx = ((ClientContext) client);
@@ -58,15 +76,6 @@ public class CompactionConfigChangeIT extends
AccumuloClusterHarness {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
final String table = getUniqueNames(1)[0];
- client.instanceOperations().setProperty(
- Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner",
- DefaultCompactionPlanner.class.getName());
- client.instanceOperations().setProperty(
- Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner.opts.executors",
- ("[{'name':'small','type':'internal','maxSize':'2M','numThreads':2},"
- +
"{'name':'medium','type':'internal','maxSize':'128M','numThreads':2},"
- +
"{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\""));
-
createTable(client, table, "cs1", 100);
ExternalCompactionTestUtils.writeData(client, table, MAX_DATA);
@@ -94,8 +103,8 @@ public class CompactionConfigChangeIT extends
AccumuloClusterHarness {
// with running compactions.
client.instanceOperations().setProperty(
Property.COMPACTION_SERVICE_PREFIX.getKey() +
"cs1.planner.opts.executors",
-
("[{'name':'little','type':'internal','maxSize':'128M','numThreads':8},"
- +
"{'name':'big','type':'internal','numThreads':2}]").replaceAll("'", "\""));
+
("[{'name':'little','type':'external','maxSize':'128M','group':'e4'},"
+ +
"{'name':'big','type':'external','group':'e5'}]").replaceAll("'", "\""));
Wait.waitFor(() -> countFiles(client, table, "F") == 0, 6);
(accumulo) branch elasticity updated: Fixes ExternalCompactionMetricsIT (#4046)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 4276d14585 Fixes ExternalCompactionMetricsIT (#4046)
4276d14585 is described below
commit 4276d14585e351c2cf1837f777e374b7d264de98
Author: Daniel Roberts
AuthorDate: Tue Dec 12 15:53:43 2023 -0500
Fixes ExternalCompactionMetricsIT (#4046)
---
.../compaction/ExternalCompactionMetricsIT.java| 35 +++---
1 file changed, 25 insertions(+), 10 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
index 7993703c30..1dde2045c9 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
@@ -20,6 +20,12 @@ package org.apache.accumulo.test.compaction;
import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP1;
import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP2;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP3;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP4;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP5;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP6;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP7;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP8;
import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact;
import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable;
import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.verify;
@@ -52,10 +58,8 @@ import
org.apache.accumulo.test.metrics.TestStatsDSink.Metric;
import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-@Disabled // ELASTICITY_TODO
public class ExternalCompactionMetricsIT extends SharedMiniClusterBase {
public static class ExternalCompactionMetricsITConfig
@@ -63,10 +67,19 @@ public class ExternalCompactionMetricsIT extends
SharedMiniClusterBase {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
coreSite) {
ExternalCompactionTestUtils.configureMiniCluster(cfg, coreSite);
- cfg.getClusterServerConfiguration().setNumDefaultCompactors(2);
+ cfg.getClusterServerConfiguration().setNumDefaultCompactors(0);
// use one tserver so that queue metrics are not spread across tservers
cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1);
+ // Override the initial state from ExternalCompactionTestUtils to not
create compactors
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP1, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP2, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP3, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP4, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP5, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP6, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP7, 0);
+ cfg.getClusterServerConfiguration().addCompactorResourceGroup(GROUP8, 0);
// Tell the server processes to use a StatsDMeterRegistry that will be
configured
// to push all metrics to the sink we started.
cfg.setProperty(Property.GENERAL_MICROMETER_ENABLED, "true");
@@ -122,7 +135,7 @@ public class ExternalCompactionMetricsIT extends
SharedMiniClusterBase {
if (shutdownTailer.get()) {
break;
}
-if (s.startsWith(MetricsProducer.METRICS_MAJC_QUEUED)) {
+if (s.startsWith(MetricsProducer.METRICS_COMPACTOR_PREFIX)) {
queueMetrics.add(TestStatsDSink.parseStatsDMetric(s));
}
}
@@ -139,8 +152,8 @@ public class ExternalCompactionMetricsIT extends
SharedMiniClusterBase {
// wait until expected number of queued are seen in metrics
while (!sawDCQ1_5 || !sawDCQ2_10) {
Metric qm = queueMetrics.take();
-sawDCQ1_5 |= match(qm, "DCQ1", "5");
-sawDCQ2_10 |= match(qm, "DCQ2", "10");
+sawDCQ1_5 |= match(qm, "dcq1", "5");
+sawDCQ2_10
(accumulo) branch main updated: Fix compaction prop log logic (#3987)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new cb1a9ac937 Fix compaction prop log logic (#3987)
cb1a9ac937 is described below
commit cb1a9ac9373b0cae311e5a67021666e14b006430
Author: Daniel Roberts
AuthorDate: Fri Dec 8 14:58:19 2023 -0500
Fix compaction prop log logic (#3987)
Fixes an incorrect logging statement from appearing when processing
`service..planner` compaction properties.
---
.../accumulo/core/util/compaction/CompactionServicesConfig.java | 8 +++-
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
index 21a7719011..5e2f233a9a 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
@@ -133,13 +133,11 @@ public class CompactionServicesConfig {
if (eprop == null || aconf.isPropertySet(eprop)) {
rateLimits.put(tokens[0],
ConfigurationTypeHelper.getFixedMemoryAsBytes(val));
}
-} else if (!(tokens.length == 2 && tokens[1].equals("planner"))) {
+} else if (tokens.length == 2 && tokens[1].equals("planner")) {
+ return; // moves to next opt
+} else {
throw new IllegalArgumentException(
"Malformed compaction service property " + prefix + prop);
-} else {
- log.warn(
- "Ignoring compaction property {} as does not match the prefix
used by the referenced planner definition",
- prop);
}
});
});
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9b62f27b820439c4989a681c9d131d40213d58ca
Merge: 54b7965e60 764f55e0c8
Author: Daniel Roberts
AuthorDate: Mon Nov 27 22:30:12 2023 +
Merge branch 'main' into elasticity
.../core/client/admin/InstanceOperations.java | 12 +++
.../accumulo/core/rpc/clients/TServerClient.java | 22 +---
.../core/spi/compaction/CompactionPlanner.java | 12 +++
.../spi/compaction/DefaultCompactionPlanner.java | 28 +++
.../server/conf/CheckCompactionConfigTest.java | 42 +++---
.../compaction/ExternalCompactionTestUtils.java| 32 -
6 files changed, 79 insertions(+), 69 deletions(-)
diff --cc
server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
index 911743f2dc,ec903814df..a8b871eb4d
---
a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
@@@ -67,12 -67,12 +67,12 @@@ public class CheckCompactionConfigTest
+
"[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+
"{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}] \n"
- + "tserver.compaction.major.service.cs2.planner="
+ + "compaction.service.cs2.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
\n"
- + "tserver.compaction.major.service.cs2.planner.opts.executors=\\\n"
+ + "compaction.service.cs2.planner.opts.executors=\\\n"
+
"[{'name':'small','type':'internal','maxSize':'16M','numThreads':7},\\\n"
+
"{'name':'medium','type':'internal','maxSize':'128M','numThreads':5},\\\n"
-+
"{'name':'large','type':'external','queue':'DCQ1'}]").replaceAll("'", "\"");
++
"{'name':'large','type':'external','group':'DCQ1'}]").replaceAll("'", "\"");
String filePath = writeToFileAndReturnPath(inputString);
@@@ -87,12 -87,12 +87,12 @@@
+
"[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
+
"{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
+ "{'name':'large','type':'internal','numThreads':2}] \n"
- + "tserver.compaction.major.service.cs2.planner="
+ + "compaction.service.cs2.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
\n"
- + "tserver.compaction.major.service.cs2.planner.opts.executors=\\\n"
+ + "compaction.service.cs2.planner.opts.executors=\\\n"
+
"[{'name':'small','type':'internal','maxSize':'16M','numThreads':7},\\\n"
+
"{'name':'medium','type':'internal','maxSize':'128M','numThreads':5},\\\n"
-+ "{'name':'large','type':'external','queue':'DCQ1'}] \n"
++ "{'name':'large','type':'external','group':'DCQ1'}] \n"
+ "compaction.service.cs3.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
\n"
+ "compaction.service.cs3.planner.opts.queues=\\\n"
@@@ -104,11 -104,11 +104,11 @@@
@Test
public void testThrowsExternalNumThreadsError() throws IOException {
- String inputString = ("tserver.compaction.major.service.cs1.planner="
+ String inputString = ("compaction.service.cs1.planner="
+ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
\n"
- + "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
+ + "compaction.service.cs1.planner.opts.executors=\\\n"
+
"[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
-+
"{'name':'medium','type':'external','maxSize':'128M','numThreads':4},\\\n"
++ "{'name':'medium','type':'external','maxSize':'128M',
'group':'DCQ1', 'numThreads':4},\\\n"
+
"{'name':'large','type':'internal','numThreads':2}]").replaceAll("'", "\"");
String expectedErrorMsg = "'numThreads' should not be specified for
external compactions";
diff --cc
test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java
index dfbe5736ab,de242583fc..6fa0fce8cd
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTe
(accumulo) branch elasticity updated (54b7965e60 -> 9b62f27b82)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 54b7965e60 Cleanup failed compaction tmp files (#3955) add 7b099dd758 Fix TServerClient.getThriftServerConnection for compactor queue names (#3984) add 764f55e0c8 Remove deprecated prefix references (#3989) new 9b62f27b82 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/admin/InstanceOperations.java | 12 +++ .../accumulo/core/rpc/clients/TServerClient.java | 22 +--- .../core/spi/compaction/CompactionPlanner.java | 12 +++ .../spi/compaction/DefaultCompactionPlanner.java | 28 +++ .../server/conf/CheckCompactionConfigTest.java | 42 +++--- .../compaction/ExternalCompactionTestUtils.java| 32 - 6 files changed, 79 insertions(+), 69 deletions(-)
(accumulo) branch main updated: Remove deprecated prefix references (#3989)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 764f55e0c8 Remove deprecated prefix references (#3989)
764f55e0c8 is described below
commit 764f55e0c8e5e3588ff0464561f808b34bf13495
Author: Daniel Roberts
AuthorDate: Mon Nov 27 17:22:40 2023 -0500
Remove deprecated prefix references (#3989)
Switch the remaining hardcoded references from
`tserver.compaction.major.service` to `compaction.service`.
---
.../core/client/admin/InstanceOperations.java | 12 +++
.../core/spi/compaction/CompactionPlanner.java | 12 +++
.../spi/compaction/DefaultCompactionPlanner.java | 28 +++
.../server/conf/CheckCompactionConfigTest.java | 42 +++---
.../test/compaction/CompactionRateLimitingIT.java | 7 ++--
.../compaction/ExternalCompactionTestUtils.java| 32 -
6 files changed, 66 insertions(+), 67 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
index 85008d4ad6..9cc3aa3658 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java
@@ -87,22 +87,22 @@ public interface InstanceOperations {
* {@code
* AccumuloClient client = getClient();
* Map acceptedProps =
client.instanceOperations().modifyProperties(currProps -> {
- * var planner =
currProps.get("tserver.compaction.major.service.default.planner");
+ * var planner =
currProps.get("compaction.service.default.planner");
* //This code will only change the compaction planner if its
currently set to default settings.
* //The endsWith() function was used to make the example
short, would be better to use equals().
* if(planner != null &&
planner.endsWith("DefaultCompactionPlanner") {
* // tservers will eventually see these compaction planner
changes and when they do they will see all of the changes at once
* currProps.keySet().removeIf(
- *prop ->
prop.startsWith("tserver.compaction.major.service.default.planner.opts."));
- *
currProps.put("tserver.compaction.major.service.default.planner","MyPlannerClassName");
- *
currProps.put("tserver.compaction.major.service.default.planner.opts.myOpt1","val1");
- *
currProps.put("tserver.compaction.major.service.default.planner.opts.myOpt2","val2");
+ *prop ->
prop.startsWith("compaction.service.default.planner.opts."));
+ *
currProps.put("compaction.service.default.planner","MyPlannerClassName");
+ *
currProps.put("compaction.service.default.planner.opts.myOpt1","val1");
+ *
currProps.put("compaction.service.default.planner.opts.myOpt2","val2");
*}
* });
*
* // Since three properties were set may want to check for the
values of all
* // three, just checking one in this example to keep it short.
- *
if("MyPlannerClassName".equals(acceptedProps.get("tserver.compaction.major.service.default.planner"))){
+ *
if("MyPlannerClassName".equals(acceptedProps.get("compaction.service.default.planner"))){
*// the compaction planner change was accepted or already
existed, so take action for that outcome
* } else {
*// the compaction planner change was not done, so take
action for that outcome
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
index f96ed6f022..222ee651ef 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
@@ -45,11 +45,10 @@ public interface CompactionPlanner {
/**
* @return The configured options. For example if the system properties
- * {@code tserver.compaction.major.service.s1.planner.opts.p1=abc}
and
- * {@code tserver.compaction.major.service.s1.planner.opts.p9=123}
were set, then this
- * map would contain {@code p1=abc} and {@code
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 347a1220a043cc7ce8573912c4765cdcff9fd307
Merge: 232f3074d9 9dd819437a
Author: Daniel Roberts
AuthorDate: Sat Nov 25 19:46:44 2023 +
Merge branch 'main' into elasticity
.../org/apache/accumulo/core/conf/Property.java| 36 +-
.../spi/compaction/DefaultCompactionPlanner.java | 181 ++---
.../compaction/CompactionPlannerInitParams.java| 9 +-
.../util/compaction/CompactionServicesConfig.java | 126 +--
.../compaction/DefaultCompactionPlannerTest.java | 166 -
.../compaction/CompactionServicesConfigTest.java | 105 ++
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 2 +-
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 1 +
pom.xml| 6 +-
.../server/compaction/CompactionJobGenerator.java | 1 +
.../server/compaction/CompactionWatcher.java | 2 +-
.../server/conf/CheckCompactionConfig.java | 3 +-
.../server/conf/CheckCompactionConfigTest.java | 47 ++-
.../accumulo/manager/upgrade/Upgrader11to12.java | 158
.../manager/upgrade/Upgrader11to12Test.java| 406 ++---
.../accumulo/shell/commands/GrepCommand.java | 2 +-
.../accumulo/shell/commands/ScanCommand.java | 2 +-
.../compaction/BadCompactionServiceConfigIT.java | 2 +-
.../test/compaction/CompactionConfigChangeIT.java | 6 +-
.../test/compaction/CompactionExecutorIT.java | 21 +-
.../apache/accumulo/test/shell/ConfigSetIT.java| 2 +-
.../org/apache/accumulo/test/util/SlowOps.java | 1 +
22 files changed, 976 insertions(+), 309 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index a3f734164e,3ab9924d8f..c371c136b5
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -581,17 -583,31 +598,22 @@@ public enum Property
"2.1.0"),
TSERV_MIGRATE_MAXCONCURRENT("tserver.migrations.concurrent.max", "1",
PropertyType.COUNT,
"The maximum number of concurrent tablet migrations for a tablet
server.", "1.3.5"),
- TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s",
PropertyType.TIMEDURATION,
- "Time a tablet server will sleep between checking which tablets need
compaction.", "1.3.5"),
+ @Deprecated(since = "3.1", forRemoval = true)
+ @ReplacedBy(property = COMPACTION_SERVICE_PREFIX)
TSERV_COMPACTION_SERVICE_PREFIX("tserver.compaction.major.service.", null,
PropertyType.PREFIX,
"Prefix for compaction services.", "2.1.0"),
+ @Deprecated(since = "3.1", forRemoval = true)
TSERV_COMPACTION_SERVICE_ROOT_PLANNER("tserver.compaction.major.service.root.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for root tablet service.", "2.1.0"),
+ @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT("tserver.compaction.major.service.root.rate.limit",
"0B",
- PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
- @Deprecated(since = "3.1", forRemoval = true)
TSERV_COMPACTION_SERVICE_ROOT_MAX_OPEN(
"tserver.compaction.major.service.root.planner.opts.maxOpen", "30",
PropertyType.COUNT,
"The maximum number of files a compaction will open.", "2.1.0"),
+ @Deprecated(since = "3.1", forRemoval = true)
TSERV_COMPACTION_SERVICE_ROOT_EXECUTORS(
"tserver.compaction.major.service.root.planner.opts.executors",
-
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':1},{'name':'huge','type':'internal','numThreads':1}]"
- .replaceAll("'", "\""),
+
"[{'name':'all','type':'external','group':'accumulo_meta'}]".replaceAll("'",
"\""),
PropertyType.STRING,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
@@@ -598,12 -615,21 +621,14 @@@
TSERV_COMPACTION_SERVICE_META_PLANNER("tserver.compaction.major.service.meta.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for metadata table.", "2.1.0"),
+ @Deprecated(since = "
