[accumulo] branch main updated: Refactor LogSorter to use config (#2191)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 908105b Refactor LogSorter to use config (#2191)
908105b is described below
commit 908105bc8fddd64aa2d5907c22ccf2a232209306
Author: Mike Miller
AuthorDate: Wed Jul 7 07:29:44 2021 -0400
Refactor LogSorter to use config (#2191)
* Refactor LogSorter writeBuffer to allow use of system configuration
when writing out sorted rfiles. This will use configured settings on the
sorted files instead of only the defaults.
---
.../java/org/apache/accumulo/tserver/log/LogSorter.java | 13 ++---
.../apache/accumulo/tserver/log/SortedLogRecoveryTest.java | 6 --
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index 293ff82..bd299af 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -32,7 +32,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@ -146,7 +145,7 @@ public class LogSorter {
// Creating a 'finished' marker will cause recovery to proceed
normally and the
// empty file will be correctly ignored downstream.
fs.mkdirs(new Path(destPath));
-writeBuffer(context, destPath, Collections.emptyList(), part++);
+writeBuffer(destPath, Collections.emptyList(), part++);
fs.create(SortedLogState.getFinishedMarkerPath(destPath)).close();
return;
}
@@ -164,10 +163,10 @@ public class LogSorter {
value.readFields(decryptingInput);
buffer.add(new Pair<>(key, value));
}
- writeBuffer(context, destPath, buffer, part++);
+ writeBuffer(destPath, buffer, part++);
buffer.clear();
} catch (EOFException ex) {
- writeBuffer(context, destPath, buffer, part++);
+ writeBuffer(destPath, buffer, part++);
break;
}
}
@@ -215,8 +214,8 @@ public class LogSorter {
}
@VisibleForTesting
- public static void writeBuffer(ServerContext context, String destPath,
- List> buffer, int part) throws IOException
{
+ void writeBuffer(String destPath, List>
buffer, int part)
+ throws IOException {
String filename = String.format("part-r-%05d.rf", part);
Path path = new Path(destPath, filename);
FileSystem fs = context.getVolumeManager().getFileSystemByPath(path);
@@ -238,7 +237,7 @@ public class LogSorter {
try (var writer = FileOperations.getInstance().newWriterBuilder()
.forFile(fullPath.toString(), fs, fs.getConf(),
context.getCryptoService())
-.withTableConfiguration(DefaultConfiguration.getInstance()).build()) {
+.withTableConfiguration(conf).build()) {
writer.startDefaultLocalityGroup();
for (var entry : keyListMap.entrySet()) {
LogFileValue val = new LogFileValue();
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 2121725..582b9bc 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@ -79,6 +79,7 @@ public class SortedLogRecoveryTest {
static final Text cq = new Text("cq");
static final Value value = new Value("value");
static ServerContext context;
+ static LogSorter logSorter;
@Rule
public TemporaryFolder tempFolder =
@@ -87,6 +88,7 @@ public class SortedLogRecoveryTest {
@Before
public void setup() {
context = EasyMock.createMock(ServerContext.class);
+logSorter = new LogSorter(context, DefaultConfiguration.getInstance());
}
static class KeyValue implements Comparable {
@@ -179,11 +181,11 @@ public class SortedLogRecoveryTest {
for (KeyValue pair : entry.getValue()) {
buffer.add(new Pair<>(pair.key, pair.value));
if (buffer.size() >= bufferSize) {
-LogSorter.writeBuffer(context, destPath, buffer, parts++);
+logSorter.writeBuffer(destPath, buffer, parts++);
buffer.clear();
}
}
-LogSorter.writeBuffer(context, destPath, buff
[accumulo] branch main updated: Catch NoNodeException in CompactRange undo (#2178)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 04a4a7b Catch NoNodeException in CompactRange undo (#2178)
04a4a7b is described below
commit 04a4a7bcb5f6e7d38829c270b46c1484e4b42650
Author: Mike Miller
AuthorDate: Thu Jun 24 06:36:22 2021 -0400
Catch NoNodeException in CompactRange undo (#2178)
* Fixes #1919
* It is possible for a compaction to run after a table is deleted so
catch the exception and print to debug, avoiding the FATE warning
---
.../manager/tableOps/compact/CompactRange.java | 33 --
1 file changed, 18 insertions(+), 15 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java
index dfa52c4..6d94c52 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java
@@ -149,24 +149,27 @@ public class CompactRange extends ManagerRepo {
ZooReaderWriter zoo = environment.getContext().getZooReaderWriter();
-zoo.mutateExisting(zTablePath, currentValue -> {
- String cvs = new String(currentValue, UTF_8);
- String[] tokens = cvs.split(",");
- long flushID = Long.parseLong(tokens[0]);
-
- String txidString = String.format("%016x", txid);
+try {
+ zoo.mutateExisting(zTablePath, currentValue -> {
+String cvs = new String(currentValue, UTF_8);
+String[] tokens = cvs.split(",");
+long flushID = Long.parseLong(tokens[0]);
- StringBuilder encodedIterators = new StringBuilder();
- for (int i = 1; i < tokens.length; i++) {
-if (tokens[i].startsWith(txidString))
- continue;
-encodedIterators.append(",");
-encodedIterators.append(tokens[i]);
- }
+String txidString = String.format("%016x", txid);
- return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
-});
+StringBuilder encodedIterators = new StringBuilder();
+for (int i = 1; i < tokens.length; i++) {
+ if (tokens[i].startsWith(txidString))
+continue;
+ encodedIterators.append(",");
+ encodedIterators.append(tokens[i]);
+}
+return (Long.toString(flushID) + encodedIterators).getBytes(UTF_8);
+ });
+} catch (NoNodeException ke) {
+ log.debug("Node for {} no longer exists.", tableId, ke);
+}
}
@Override
[accumulo] branch main updated: Adds tests to LogFileKeyTest (#2177)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 89b411c Adds tests to LogFileKeyTest (#2177)
89b411c is described below
commit 89b411c391675cb3f1faed83f2dd4451b2ac6d12
Author: Mike Miller
AuthorDate: Wed Jun 23 09:57:31 2021 -0400
Adds tests to LogFileKeyTest (#2177)
---
.../apache/accumulo/tserver/logger/LogFileKey.java | 15 ++-
.../accumulo/tserver/log/LogFileKeyTest.java | 49 ++
2 files changed, 62 insertions(+), 2 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
index cf5dc21..b860be5 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
@@ -28,6 +28,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.util.Arrays;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
@@ -36,10 +37,13 @@ import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
public class LogFileKey implements WritableComparable {
+ private static final Logger log = LoggerFactory.getLogger(LogFileKey.class);
public LogEvents event;
public String filename = null;
@@ -258,7 +262,9 @@ public class LogFileKey implements
WritableComparable {
/**
* Format the row using 13 bytes encoded to allow proper sorting of the
RFile Key. The highest
- * byte is for the event number, 4 bytes for the tabletId and 8 bytes for
the sequence long.
+ * byte is for the event number, 4 bytes for the tabletId and 8 bytes for
the sequence long. The
+ * conversions of integer to byte[] and long to byte[] is similar to what
DataOutputStream does
+ * for writeInt() and writeLong()
*/
private byte[] formatRow(int tabletId, long seq) {
byte eventNum = getEventByte();
@@ -283,11 +289,14 @@ public class LogFileKey implements
WritableComparable {
row[10] = (byte) ((seq >>> 16) & mask);
row[11] = (byte) ((seq >>> 8) & mask);
row[12] = (byte) (seq & mask);
+
+log.trace("Convert {} {} {} to row {}", event, tabletId, seq,
Arrays.toString(row));
return row;
}
/**
- * Extract the tabletId integer from the byte encoded Row.
+ * Extract the tabletId integer from the byte encoded Row. Similar to what
DataInputStream does
+ * for readInt()
*/
private static int getTabletId(byte[] row) {
int mask = 0xff; // use a mask of int type to convert byte to int without
sign extension
@@ -335,6 +344,8 @@ public class LogFileKey implements
WritableComparable {
if (eventType(logFileKey.event) != rowParts[0]) {
throw new AssertionError("Event in row differs from column family. Key:
" + key);
}
+log.trace("From row {} get {} {} {}", Arrays.toString(rowParts),
logFileKey.event,
+logFileKey.tabletId, logFileKey.seq);
// handle special cases of what is stored in the qualifier
switch (logFileKey.event) {
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogFileKeyTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogFileKeyTest.java
index 72cc339..0f04aca 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogFileKeyTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogFileKeyTest.java
@@ -20,12 +20,17 @@ package org.apache.accumulo.tserver.log;
import static org.junit.Assert.assertEquals;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.tserver.logger.LogEvents;
import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.hadoop.io.Text;
import org.junit.Test;
public class LogFileKeyTest {
@@ -98,4 +103,48 @@ public class LogFileKeyTest {
}
}
+
+ private void testToFromKey(int tabletId, long seq) throws IOException {
+var logFileKey = nk(LogEvents.DEFINE_TABLET, tabletId, seq);
+logFileKey.tablet = new KeyExtent(TableId.of("5"), new Text("b"), null);
+Key k = logFileKey.toKey();
+
+var converted = LogFileKey.fromKey(k);
+assertEquals("Failed
[accumulo] branch main updated: Fix row encoding/decoding in LogFileKey (#2176)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new b8262a4 Fix row encoding/decoding in LogFileKey (#2176)
b8262a4 is described below
commit b8262a424db92729c89d6870ba5973386b9e3ea1
Author: Christopher Tubbs
AuthorDate: Wed Jun 23 08:46:15 2021 -0400
Fix row encoding/decoding in LogFileKey (#2176)
Fix bitwise operations to encode and decode the tabletId and sequence
number to/from a row
This fixes #2173
---
.../apache/accumulo/tserver/logger/LogFileKey.java | 54 +-
1 file changed, 31 insertions(+), 23 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
index bf08ca4..cf5dc21 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogFileKey.java
@@ -245,8 +245,8 @@ public class LogFileKey implements
WritableComparable {
* byte in the row.
*/
private byte getEventByte() {
-int evenTypeInteger = eventType(event);
-return (byte) (evenTypeInteger & 0xff);
+int eventTypeInteger = eventType(event);
+return (byte) (eventTypeInteger & 0xff);
}
/**
@@ -269,19 +269,20 @@ public class LogFileKey implements
WritableComparable {
// encode the signed integer so negatives will sort properly for tabletId
int encodedTabletId = tabletId ^ 0x8000;
+int mask = 0xff; // use a mask of int type to truncate the selected 8 bits
row[0] = eventNum;
-row[1] = (byte) ((encodedTabletId >>> 24) & 0xff);
-row[2] = (byte) ((encodedTabletId >>> 16) & 0xff);
-row[3] = (byte) ((encodedTabletId >>> 8) & 0xff);
-row[4] = (byte) (encodedTabletId & 0xff);
-row[5] = (byte) (seq >>> 56);
-row[6] = (byte) (seq >>> 48);
-row[7] = (byte) (seq >>> 40);
-row[8] = (byte) (seq >>> 32);
-row[9] = (byte) (seq >>> 24);
-row[10] = (byte) (seq >>> 16);
-row[11] = (byte) (seq >>> 8);
-row[12] = (byte) (seq); // >>> 0
+row[1] = (byte) ((encodedTabletId >>> 24) & mask);
+row[2] = (byte) ((encodedTabletId >>> 16) & mask);
+row[3] = (byte) ((encodedTabletId >>> 8) & mask);
+row[4] = (byte) (encodedTabletId & mask);
+row[5] = (byte) ((seq >>> 56) & mask);
+row[6] = (byte) ((seq >>> 48) & mask);
+row[7] = (byte) ((seq >>> 40) & mask);
+row[8] = (byte) ((seq >>> 32) & mask);
+row[9] = (byte) ((seq >>> 24) & mask);
+row[10] = (byte) ((seq >>> 16) & mask);
+row[11] = (byte) ((seq >>> 8) & mask);
+row[12] = (byte) (seq & mask);
return row;
}
@@ -289,7 +290,13 @@ public class LogFileKey implements
WritableComparable {
* Extract the tabletId integer from the byte encoded Row.
*/
private static int getTabletId(byte[] row) {
-int encoded = ((row[1] << 24) + (row[2] << 16) + (row[3] << 8) + row[4]);
+int mask = 0xff; // use a mask of int type to convert byte to int without
sign extension
+// @formatter:off
+int encoded = (row[1] & mask) << 24 |
+ (row[2] & mask) << 16 |
+ (row[3] & mask) << 8 |
+ (row[4] & mask);
+// @formatter:on
return encoded ^ 0x8000;
}
@@ -297,15 +304,16 @@ public class LogFileKey implements
WritableComparable {
* Extract the sequence long from the byte encoded Row.
*/
private static long getSequence(byte[] row) {
+long mask = 0xff; // use a mask of long type to convert byte to long
without sign extension
// @formatter:off
-return (((long) row[5] << 56) +
-((long) (row[6] & 255) << 48) +
-((long) (row[7] & 255) << 40) +
-((long) (row[8] & 255) << 32) +
-((long) (row[9] & 255) << 24) +
-((row[10] & 255) << 16) +
-((row[11] & 255) << 8) +
-((row[12] & 255)));
+return (row[5] & mask) << 56 |
+ (row[6] & mask) << 48 |
+ (row[7] & mask) << 40 |
+ (row[8] & mask) << 32 |
+ (row[9] & mask) << 24 |
+ (row[10] & mask) << 16 |
+ (row[11] & mask) << 8 |
+ (row[12] & mask);
// @formatter:on
}
[accumulo] branch main updated: Make sorted recovery write to RFiles (#2117)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3e765b1 Make sorted recovery write to RFiles (#2117)
3e765b1 is described below
commit 3e765b1f3b487092206cee036097820c37b7907e
Author: Mike Miller
AuthorDate: Mon Jun 21 09:53:33 2021 -0400
Make sorted recovery write to RFiles (#2117)
* Change LogSorter and SortedLogRecovery to write/read RFile instead of
MapFile
* Rewrite LogSorter writeBuffer method and make static to test
* Create toKey() and fromKey() methods in LogFileKey for converting WAL keys
* Create toValue() and FromValue() methods in LogFileValue for converting
WAL values
* Byte encode the rows of the Key to sort WAL events properly
* Make RecoveryLogsIterator convert Key Value pairs on next()
Co-authored-by: Keith Turner
Co-authored-by: Christopher Tubbs
---
.../org/apache/accumulo/tserver/TabletServer.java | 6 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 64 +---
.../accumulo/tserver/log/RecoveryLogReader.java| 2 +-
.../accumulo/tserver/log/RecoveryLogsIterator.java | 124 +++
.../accumulo/tserver/log/SortedLogRecovery.java| 63
.../accumulo/tserver/log/TabletServerLogger.java | 9 +-
.../apache/accumulo/tserver/logger/LogFileKey.java | 167 +
.../accumulo/tserver/logger/LogFileValue.java | 28
.../tserver/log/SortedLogRecoveryTest.java | 79 +++---
.../tserver/log/TestUpgradePathForWALogs.java | 3 +
10 files changed, 436 insertions(+), 109 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 6887f4a..2ab89fa 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -1133,7 +1133,7 @@ public class TabletServer extends AbstractServer {
public void recover(VolumeManager fs, KeyExtent extent, List
logEntries,
Set tabletFiles, MutationReceiver mutationReceiver) throws
IOException {
-List recoveryLogs = new ArrayList<>();
+List recoveryDirs = new ArrayList<>();
List sorted = new ArrayList<>(logEntries);
sorted.sort((e1, e2) -> (int) (e1.timestamp - e2.timestamp));
for (LogEntry entry : sorted) {
@@ -1148,9 +1148,9 @@ public class TabletServer extends AbstractServer {
throw new IOException(
"Unable to find recovery files for extent " + extent + " logEntry:
" + entry);
}
- recoveryLogs.add(recovery);
+ recoveryDirs.add(recovery);
}
-logger.recover(fs, extent, recoveryLogs, tabletFiles, mutationReceiver);
+logger.recover(getContext(), extent, recoveryDirs, tabletFiles,
mutationReceiver);
}
public int createLogId() {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index 98fe935..293ff82 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -23,16 +23,20 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.master.thrift.RecoveryStatus;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.threads.ThreadPools;
@@ -47,11 +51,12 @@ import org.apache.accumulo.tserver.logger.LogFileValue;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.MapFile;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
public class LogSorter {
private static final Logger log = LoggerFactory.getLogger(LogSorter.class);
@@ -141,7 +146,7 @@ public class LogSorter {
// Creating a 'finished' marker will cause recovery to proceed
[accumulo] branch main updated: Add test and fix for bug in KeyBuilder (#2170)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1fecdd3 Add test and fix for bug in KeyBuilder (#2170)
1fecdd3 is described below
commit 1fecdd378f2bc15955baa371ac9bb706bcc94dfc
Author: Mike Miller
AuthorDate: Thu Jun 17 09:40:49 2021 -0400
Add test and fix for bug in KeyBuilder (#2170)
* Replace error prone CharsetEncoder with toString() and getBytes()
---
.../main/java/org/apache/accumulo/core/data/KeyBuilder.java | 13 +
.../java/org/apache/accumulo/core/data/KeyBuilderTest.java | 10 ++
2 files changed, 11 insertions(+), 12 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/data/KeyBuilder.java
b/core/src/main/java/org/apache/accumulo/core/data/KeyBuilder.java
index 100a8cc..1d625b8 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyBuilder.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyBuilder.java
@@ -20,11 +20,6 @@ package org.apache.accumulo.core.data;
import static java.nio.charset.StandardCharsets.UTF_8;
-import java.nio.CharBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CodingErrorAction;
-
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
@@ -326,13 +321,7 @@ public class KeyBuilder {
}
private byte[] encodeCharSequence(CharSequence chars) {
- CharsetEncoder encoder =
UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPORT)
- .onUnmappableCharacter(CodingErrorAction.REPORT);
- try {
-return encoder.encode(CharBuffer.wrap(chars)).array();
- } catch (CharacterCodingException ex) {
-throw new RuntimeException("KeyBuilder supports only CharSequences
encoded in UTF-8", ex);
- }
+ return chars.toString().getBytes(UTF_8);
}
@Override
diff --git
a/core/src/test/java/org/apache/accumulo/core/data/KeyBuilderTest.java
b/core/src/test/java/org/apache/accumulo/core/data/KeyBuilderTest.java
index 8f795c5..70f02e5 100644
--- a/core/src/test/java/org/apache/accumulo/core/data/KeyBuilderTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/KeyBuilderTest.java
@@ -305,4 +305,14 @@ public class KeyBuilderTest {
keyExpected.setDeleted(false);
assertEquals(keyExpected, keyBuilt);
}
+
+ /**
+ * Tests bug where a String of 10 chars or longer was being encoded
incorrectly.
+ */
+ @Test
+ public void test10CharactersBug() {
+Key keyBuilt1 = Key.builder().row(rowText).family("1234567890").build();
+Key keyBuilt2 = Key.builder().row(rowText).family(new
Text("1234567890")).build();
+assertEquals(keyBuilt1, keyBuilt2);
+ }
}
[accumulo] branch main updated: Fix crypto in SplitLarge utility (#2145)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 512325e Fix crypto in SplitLarge utility (#2145)
512325e is described below
commit 512325e1ab14351df7722ca1956b14c42fff90d7
Author: Mike Miller
AuthorDate: Fri Jun 4 13:31:01 2021 -0400
Fix crypto in SplitLarge utility (#2145)
* The option that was added in 2.0 for the encryption only included the
name of the service. You also need a way to pass in other options for it to
work. This change just simplifies that by removing the option altogether and
reading the config from classpath, similar to other utilities.
---
.../org/apache/accumulo/core/file/rfile/SplitLarge.java| 14 +-
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
index f0ca7da..580c68d 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
@@ -21,10 +21,8 @@ package org.apache.accumulo.core.file.rfile;
import java.util.ArrayList;
import java.util.List;
-import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.cli.ConfigOpts;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.data.Key;
@@ -46,12 +44,10 @@ import com.beust.jcommander.Parameter;
*/
public class SplitLarge {
- static class Opts extends Help {
+ static class Opts extends ConfigOpts {
@Parameter(names = "-m",
description = "the maximum size of the key/value pair to shunt to the
small file")
long maxSize = 10 * 1024 * 1024;
-@Parameter(names = "-crypto", description = "the class to perform
encryption/decryption")
-String cryptoClass = Property.INSTANCE_CRYPTO_SERVICE.getDefaultValue();
@Parameter(description = " { ... }")
List files = new ArrayList<>();
}
@@ -63,9 +59,9 @@ public class SplitLarge {
opts.parseArgs(SplitLarge.class.getName(), args);
for (String file : opts.files) {
- AccumuloConfiguration aconf = DefaultConfiguration.getInstance();
- CryptoService cryptoService =
ConfigurationTypeHelper.getClassInstance(null, opts.cryptoClass,
- CryptoService.class, CryptoServiceFactory.newDefaultInstance());
+ AccumuloConfiguration aconf = opts.getSiteConfiguration();
+ CryptoService cryptoService =
+ CryptoServiceFactory.newInstance(aconf,
CryptoServiceFactory.ClassloaderType.JAVA);
Path path = new Path(file);
CachableBuilder cb =
new CachableBuilder().fsPath(fs,
path).conf(conf).cryptoService(cryptoService);
[accumulo] branch main updated (af2d2d6 -> b15fbf3)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from af2d2d6 Improve LocalityGroupUtil.java (#2112) add b15fbf3 Move logic in LogSorter outside sort method (#2113) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/tserver/log/LogSorter.java | 119 ++--- .../tserver/log/TestUpgradePathForWALogs.java | 97 + .../tserver/src/test/resources/walog-from-20.walog | Bin 0 -> 982 bytes 3 files changed, 108 insertions(+), 108 deletions(-) create mode 100755 server/tserver/src/test/resources/walog-from-20.walog
[accumulo-website] branch main updated: Update people.md (#283)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new 8f0de12 Update people.md (#283) 8f0de12 is described below commit 8f0de12ec2f3190c0fbedcb0ff64aa41884f559a Author: nicolasalarconrapela AuthorDate: Thu May 20 17:20:57 2021 +0300 Update people.md (#283) --- pages/people.md | 1 + 1 file changed, 1 insertion(+) diff --git a/pages/people.md b/pages/people.md index 7c0cb80..80e142c 100644 --- a/pages/people.md +++ b/pages/people.md @@ -147,6 +147,7 @@ GitHub also has a [contributor list][github-contributors] based on commits. | Mike Fagan | [Arcus Research][ARCUS] | [MT][MT] | | Morgan Haskel | | | | Nguessan Kouame | | | +| Nicolás Alarcón R. | | [CEST][CEST] | | Oren Falkowitz | [sqrrl][SQRRL] | [ET][ET] | | Phil Eberhardt | [sqrrl][SQRRL] | [ET][ET] | | Philip Young| | |
[accumulo] branch main updated: Fix property JSON strings (#2109)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 763cf4f Fix property JSON strings (#2109)
763cf4f is described below
commit 763cf4fae5d93aff96a06458f98dfe7296b5b882
Author: Mike Miller
AuthorDate: Fri May 14 14:10:01 2021 -0400
Fix property JSON strings (#2109)
* Apply replaceAll for quotes on all parts of the default JSON values in
Property.java. Closes #2105
---
.../main/java/org/apache/accumulo/core/conf/Property.java | 13 ++---
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 30b9ce1..1b14a8c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -442,8 +442,8 @@ public enum Property {
"The maximum number of files a compaction will open"),
TSERV_COMPACTION_SERVICE_ROOT_EXECUTORS(
"tserver.compaction.major.service.root.planner.opts.executors",
- "[{'name':'small','maxSize':'32M','numThreads':1},"
- + "{'name':'huge','numThreads':1}]".replaceAll("'", "\""),
+
("[{'name':'small','maxSize':'32M','numThreads':1},{'name':'huge','numThreads':1}]")
+ .replaceAll("'", "\""),
PropertyType.STRING,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} "),
TSERV_COMPACTION_SERVICE_META_PLANNER("tserver.compaction.major.service.meta.planner",
@@ -458,8 +458,8 @@ public enum Property {
"The maximum number of files a compaction will open"),
TSERV_COMPACTION_SERVICE_META_EXECUTORS(
"tserver.compaction.major.service.meta.planner.opts.executors",
- "[{'name':'small','maxSize':'32M','numThreads':2},"
- + "{'name':'huge','numThreads':2}]".replaceAll("'", "\""),
+
("[{'name':'small','maxSize':'32M','numThreads':2},{'name':'huge','numThreads':2}]")
+ .replaceAll("'", "\""),
PropertyType.STRING,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} "),
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
@@ -474,9 +474,8 @@ public enum Property {
"The maximum number of files a compaction will open"),
TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS(
"tserver.compaction.major.service.default.planner.opts.executors",
- "[{'name':'small','maxSize':'32M','numThreads':2},"
- + "{'name':'medium','maxSize':'128M','numThreads':2},"
- + "{'name':'large','numThreads':2}]".replaceAll("'", "\""),
+
("[{'name':'small','maxSize':'32M','numThreads':2},{'name':'medium','maxSize':'128M','numThreads':2},{'name':'large','numThreads':2}]")
+ .replaceAll("'", "\""),
PropertyType.STRING,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} "),
@Deprecated(since = "2.1.0", forRemoval = true)
[accumulo] branch main updated (a28b27a -> 3c039c7)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from a28b27a Add extent to metadata update failure logs (#2086) add 3c039c7 Add option to disable encryption to AESCryptoService (#2064) No new revisions were added by this update. Summary of changes: .../accumulo/core/spi/crypto/AESCryptoService.java | 25 ++--- .../apache/accumulo/core/crypto/CryptoTest.java| 41 ++ .../apache/accumulo/core/file/rfile/RFileTest.java | 5 +++ 3 files changed, 66 insertions(+), 5 deletions(-)
[accumulo] branch main updated: Fix #2070 address ShellServerIT test failures (#2087)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a0af230 Fix #2070 address ShellServerIT test failures (#2087)
a0af230 is described below
commit a0af230ab65039ba4eda5e2d6ee415f89f08914c
Author: EdColeman
AuthorDate: Mon May 10 12:46:29 2021 -0400
Fix #2070 address ShellServerIT test failures (#2087)
- address simple checkstyle errors
- use unique table and namespace names
- Add test for constraints jar built with 2_1
---
.../org/apache/accumulo/test/ShellServerIT.java| 465 +++--
test/src/main/resources/FooConstraint_2_1.jar | Bin 0 -> 2018 bytes
2 files changed, 252 insertions(+), 213 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index bd166af..081e984 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -375,7 +375,8 @@ public class ShellServerIT extends SharedMiniClusterBase {
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path
provided by test")
@Test
public void exporttableImporttable() throws Exception {
-final String table = name.getMethodName(), table2 = table + "2";
+final String table = getUniqueNames(1)[0];
+final String table2 = table + "2";
// exporttable / importtable
ts.exec("createtable " + table + " -evc", true);
@@ -452,7 +453,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void setscaniterDeletescaniter() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
// setscaniter, deletescaniter
ts.exec("createtable " + table);
@@ -482,7 +483,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void egrep() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
// egrep
ts.exec("createtable " + table);
@@ -494,7 +495,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void du() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
// create and delete a table so we get out of a table context in the shell
ts.exec("notable", true);
@@ -513,7 +514,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
String o = ts.output.get();
// for some reason, there's a bit of fluctuation
assertTrue("Output did not match regex: '" + o + "'",
-o.matches(".*[1-9][0-9][0-9]\\s\\[" + table + "\\]\\n"));
+o.matches(".*[1-9][0-9][0-9]\\s\\[" + table + "]\\n"));
ts.exec("deletetable -f " + table);
}
@@ -535,7 +536,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void user() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
final boolean kerberosEnabled = getToken() instanceof KerberosToken;
// createuser, deleteuser, user, users, droptable, grant, revoke
@@ -577,7 +578,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void durability() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
ts.exec("createtable " + table);
ts.exec("insert -d none a cf cq randomGunkaASDFWEAQRd");
ts.exec("insert -d foo a cf cq2 2", false, "foo", true);
@@ -587,7 +588,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void iter() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
// setshelliter, listshelliter, deleteshelliter
ts.exec("createtable " + table);
@@ -637,9 +638,10 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Test
public void setIterOptionPrompt() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- String tableName = name.getMethodName();
+ final String[] tableNames = getUniqueNames(4);
+ final String tableName0 = tableNames[0];
- ts.exec("createtable " + tableName);
+ ts.exec("createtable " + tableName0);
ts.input.set("\n\n");
// Setting a non-optiondescriber with no name should fail
ts.exec("setiter -scan -class " + COLUMN_FAMILY_COUNTER_ITERATOR + "
[accumulo] branch main updated: Create MockServerContext for use in unit tests (#2083)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6a0a5ee Create MockServerContext for use in unit tests (#2083)
6a0a5ee is described below
commit 6a0a5ee30788f9f606166b8391b22c23d65071dc
Author: Mike Miller
AuthorDate: Thu May 6 15:13:54 2021 -0400
Create MockServerContext for use in unit tests (#2083)
---
.../apache/accumulo/server/MockServerContext.java | 55 ++
.../accumulo/server/client/BulkImporterTest.java | 11 +
.../server/conf/NamespaceConfigurationTest.java| 10 +---
.../conf/ServerConfigurationFactoryTest.java | 16 +--
.../server/conf/TableConfigurationTest.java| 10 +---
.../server/conf/ZooConfigurationFactoryTest.java | 7 +--
.../master/balancer/TableLoadBalancerTest.java | 10 +---
.../server/problems/ProblemReportTest.java | 4 +-
.../security/handler/ZKAuthenticatorTest.java | 7 ++-
9 files changed, 72 insertions(+), 58 deletions(-)
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java
b/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java
new file mode 100644
index 000..64694e8
--- /dev/null
+++
b/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server;
+
+import static org.easymock.EasyMock.expect;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.crypto.CryptoServiceFactory;
+import org.easymock.EasyMock;
+
+/**
+ * Get a generic mocked ServerContext with junk for testing.
+ */
+public class MockServerContext {
+
+ public static ServerContext get() {
+ServerContext sc = EasyMock.createMock(ServerContext.class);
+ConfigurationCopy conf = new
ConfigurationCopy(DefaultConfiguration.getInstance());
+conf.set(Property.INSTANCE_VOLUMES, "file:///");
+expect(sc.getConfiguration()).andReturn(conf).anyTimes();
+
expect(sc.getCryptoService()).andReturn(CryptoServiceFactory.newDefaultInstance()).anyTimes();
+expect(sc.getProperties()).andReturn(new Properties()).anyTimes();
+return sc;
+ }
+
+ public static ServerContext getWithZK(String instanceID, String zk, int
zkTimeout) {
+var sc = get();
+expect(sc.getZooKeeperRoot()).andReturn("/accumulo/" +
instanceID).anyTimes();
+expect(sc.getInstanceID()).andReturn(instanceID).anyTimes();
+expect(sc.getZooKeepers()).andReturn(zk).anyTimes();
+expect(sc.getZooKeepersSessionTimeOut()).andReturn(zkTimeout).anyTimes();
+return sc;
+ }
+
+}
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
b/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
index 566596d..b0d17be 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/client/BulkImporterTest.java
@@ -31,9 +31,6 @@ import java.util.TreeSet;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.TabletLocator;
import org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation;
-import org.apache.accumulo.core.conf.ConfigurationCopy;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@ -43,6 +40,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.server.MockServerContext;
import org.apach
[accumulo] branch main updated: Fix for Constraint API (#2074)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6094a53 Fix for Constraint API (#2074)
6094a53 is described below
commit 6094a537421f7b2d328c8d1d016a2bf830743d4c
Author: Mike Miller
AuthorDate: Mon May 3 17:13:04 2021 -0400
Fix for Constraint API (#2074)
* Add method to deprecated API to prevent errors
* Add test to DeprecatedConstraintExtendTest
---
.../accumulo/core/constraints/Constraint.java | 11 +++
.../DeprecatedConstraintExtendTest.java| 38 --
2 files changed, 47 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
b/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
index 3414970..b24d613 100644
--- a/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
+++ b/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
@@ -84,4 +84,15 @@ public interface Constraint extends
org.apache.accumulo.core.data.constraints.Co
* @return list of violation codes, or null if none
*/
List check(Environment env, Mutation mutation);
+
+ /**
+ * Implemented for backwards compatibility.
+ *
+ * @since 2.1.0
+ */
+ @Override
+ default List
check(org.apache.accumulo.core.data.constraints.Constraint.Environment env,
+ Mutation mutation) {
+return check((Environment) env, mutation);
+ }
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
b/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
index aa7ef99..dac943d 100644
---
a/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
@@ -40,13 +40,12 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressWarnings("deprecation")
public class DeprecatedConstraintExtendTest {
- Constraint constraint = new MinKeySizeConstraint();
-
byte[] min = new byte[1024];
byte[] oversized = new byte[1048577];
@Test
public void testMinKeySizeConstraint() {
+Constraint constraint = new MinKeySizeConstraint();
// pass constraints
Mutation m = new Mutation(min);
@@ -67,6 +66,20 @@ public class DeprecatedConstraintExtendTest {
constraint.check(null, m));
}
+ @Test
+ public void testFoo() {
+FooConstraint fc = new FooConstraint();
+// pass constraints
+Mutation m = new Mutation("blah");
+m.put("colf", "colq", new Value(new byte[] {}));
+assertEquals(null, fc.check(null, m));
+
+// test fail constraint
+m = new Mutation("foo");
+m.put("colf", "colq", new Value(new byte[] {}));
+assertEquals(Collections.singletonList(Short.valueOf("1")), fc.check(null,
m));
+ }
+
/**
* Limit the size of 1mb but also a minimum of 1KB
*/
@@ -94,4 +107,25 @@ public class DeprecatedConstraintExtendTest {
return violations;
}
}
+
+ /**
+ * Test previously defined constraint.
+ */
+ public class FooConstraint implements Constraint {
+public String getViolationDescription(short violationCode) {
+ switch (violationCode) {
+case 1:
+ return "Contains foo";
+ }
+ throw new IllegalArgumentException();
+}
+
+public List check(Constraint.Environment env, Mutation mutation) {
+ if (new String(mutation.getRow()).contains("foo")) {
+return Collections.singletonList(Short.valueOf("1"));
+ }
+ return null;
+}
+ }
+
}
[accumulo] branch main updated: Fix redundant TableId creation in TableLoadBalancer (#2060)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4213d76 Fix redundant TableId creation in TableLoadBalancer (#2060)
4213d76 is described below
commit 4213d76c2b5fcbeeb4163db04c1fb8b7226cfe22
Author: Mike Miller
AuthorDate: Fri Apr 30 09:34:51 2021 -0400
Fix redundant TableId creation in TableLoadBalancer (#2060)
---
.../org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java| 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java
b/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java
index f046984..02aef99 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java
@@ -108,10 +108,8 @@ public class TableLoadBalancer implements TabletBalancer {
public void getAssignments(AssignmentParameters params) {
// separate the unassigned into tables
Map> groupedUnassigned = new
HashMap<>();
-params.unassignedTablets()
-.forEach((tid, lastTserver) -> groupedUnassigned
-.computeIfAbsent(TableId.of(tid.getTable().canonical()), k -> new
HashMap<>())
-.put(tid, lastTserver));
+params.unassignedTablets().forEach((tid, lastTserver) -> groupedUnassigned
+.computeIfAbsent(tid.getTable(), k -> new HashMap<>()).put(tid,
lastTserver));
for (Entry> e :
groupedUnassigned.entrySet()) {
Map newAssignments = new HashMap<>();
getBalancerForTable(e.getKey()).getAssignments(
[accumulo-website] branch asf-site updated (c611f61 -> a0db284)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from c611f61 Automatic Site Publish by Buildbot add 9c2604e Automatic Site Publish by Buildbot add a0db284 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 ++-- output/how-to-contribute/index.html | 2 +- output/people/index.html| 5 + 3 files changed, 8 insertions(+), 3 deletions(-)
[accumulo-website] branch main updated: Add build command to pages/how-to-contribute.md (#276)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new b670b9a Add build command to pages/how-to-contribute.md (#276) b670b9a is described below commit b670b9af0b054629db930c340fa747814e5829aa Author: Mike Miller AuthorDate: Thu Apr 29 09:13:33 2021 -0400 Add build command to pages/how-to-contribute.md (#276) --- pages/how-to-contribute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/how-to-contribute.md b/pages/how-to-contribute.md index 7537bae..7857232 100644 --- a/pages/how-to-contribute.md +++ b/pages/how-to-contribute.md @@ -40,7 +40,7 @@ Any questions/ideas don't hesitate to [contact us][contact]. git checkout -b accumulo-4321 ``` 1. Do work and commit to your branch. You can reference [this link][messages] for a guide on how to write good commit log messages. -1. Ensure you works satisfies the guidelines laid out in the `CONTRIBUTING.md` file. +1. Ensure your branch builds cleanly using the command: ```mvn clean verify -DskipITs``` 1. If needed, squash to the minimum number of commits. For help on squashing commits, see this [tutorial][squash-tutorial] or [StackOverflow answer][squash-stack]. 1. [Push] your branch to your fork. ```
[accumulo] branch main updated: Migrate Constraints to API (#1985)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d3b41fc Migrate Constraints to API (#1985)
d3b41fc is described below
commit d3b41fc0ac5da70aab86e6fb95852353b9a167aa
Author: Mike Miller
AuthorDate: Wed Apr 28 14:25:18 2021 -0400
Migrate Constraints to API (#1985)
* Create new Constraint API and equivalent classes under the
o.a.a.core.data.constraints package to become public API
* Deprecated Constraint and classes under o.a.a.core.constraints package
and make the old classes extend the new API
* Migrate tests to go along with API changes
* Create DeprecatedConstraintExtendTest to test user extending old API
* Add DefaultKeySizeConstraintTest to reuseForks exceptions
---
core/pom.xml | 2 +
.../core/clientImpl/NamespaceOperationsImpl.java | 2 +-
.../core/clientImpl/TableOperationsImpl.java | 2 +-
.../apache/accumulo/core/conf/IterConfigUtil.java | 2 +-
.../org/apache/accumulo/core/conf/Property.java| 2 +-
.../accumulo/core/constraints/Constraint.java | 26 ++
.../core/constraints/DefaultKeySizeConstraint.java | 13 ++-
.../core/constraints/NoDeleteConstraint.java | 12 ++-
.../core/constraints/VisibilityConstraint.java | 13 ++-
.../core/{ => data}/constraints/Constraint.java| 19 +++--
.../constraints/DefaultKeySizeConstraint.java | 4 +-
.../{ => data}/constraints/NoDeleteConstraint.java | 4 +-
.../constraints/VisibilityConstraint.java | 4 +-
.../constraints/DefaultKeySizeConstraintTest.java | 1 +
.../DeprecatedConstraintExtendTest.java| 97 ++
.../core/constraints/NoDeleteConstraintTest.java | 1 +
.../VisibilityConstraintTest.java | 11 +--
.../constraints/DefaultKeySizeConstraintTest.java | 7 +-
.../constraints/NoDeleteConstraintTest.java| 2 +-
.../constraints}/VisibilityConstraintTest.java | 7 +-
.../server/constraints/MetadataConstraints.java| 2 +-
.../server/constraints/SystemEnvironment.java | 2 +-
.../server/metadata/RootTabletMutatorImpl.java | 15 +++-
.../accumulo/tserver/TservConstraintEnv.java | 21 +++--
.../tserver/constraints/ConstraintChecker.java | 20 -
.../tserver/constraints/SystemConstraint.java | 2 +-
.../constraints/UnsatisfiableConstraint.java | 2 +-
.../tserver/constraints/ConstraintCheckerTest.java | 34
.../accumulo/shell/commands/ConstraintCommand.java | 2 +-
.../shell/commands/CreateTableCommand.java | 2 +-
.../apache/accumulo/test/TableOperationsIT.java| 2 +-
.../test/constraints/AlphaNumKeyConstraint.java| 2 +-
.../accumulo/test/constraints/MaxMutationSize.java | 2 +-
.../test/constraints/NumericValueConstraint.java | 2 +-
.../accumulo/test/functional/SlowConstraint.java | 2 +-
35 files changed, 252 insertions(+), 91 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 36d9dc6..e546886 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -206,6 +206,7 @@
**/IdleRatioScanPrioritizerTest.java
**/TestCfCqSlice.java
**/TestCfCqSliceSeekingFilter.java
+
**/core/constraints/DefaultKeySizeConstraintTest.java
@@ -223,6 +224,7 @@
**/IdleRatioScanPrioritizerTest.java
**/TestCfCqSlice.java
**/TestCfCqSliceSeekingFilter.java
+
**/core/constraints/DefaultKeySizeConstraintTest.java
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
index 126b977..979d891 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
@@ -49,8 +49,8 @@ import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode;
import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
import
org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException;
-import org.apache.accumulo.core.constraints.Constraint;
import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.constraints.Constraint;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.manager.thrift.FateOperation;
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/
[accumulo] branch main updated: Made private fields in Violations.CVSKey final (#2036)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9afab2c Made private fields in Violations.CVSKey final (#2036)
9afab2c is described below
commit 9afab2c1b931d100ed6bad5cfe028c28da944427
Author: alimustafashah
AuthorDate: Tue Apr 27 18:08:12 2021 +0500
Made private fields in Violations.CVSKey final (#2036)
* Closes #2033
---
.../main/java/org/apache/accumulo/core/constraints/Violations.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/constraints/Violations.java
b/core/src/main/java/org/apache/accumulo/core/constraints/Violations.java
index 4e8dc5b..821de1e 100644
--- a/core/src/main/java/org/apache/accumulo/core/constraints/Violations.java
+++ b/core/src/main/java/org/apache/accumulo/core/constraints/Violations.java
@@ -34,8 +34,8 @@ import
org.apache.accumulo.core.data.ConstraintViolationSummary;
public class Violations {
private static class CVSKey {
-private String className;
-private short vcode;
+final private String className;
+final private short vcode;
CVSKey(ConstraintViolationSummary cvs) {
this.className = cvs.constrainClass;
[accumulo-testing] branch main updated: Update pom to Java 11 (#138)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 64c5e17 Update pom to Java 11 (#138)
64c5e17 is described below
commit 64c5e17589258dcfd2e66b041abc565b4c1afce4
Author: Mike Miller
AuthorDate: Wed Apr 14 07:53:38 2021 -0400
Update pom to Java 11 (#138)
* Update pom to Java 11
* Remove obsoleted profile
* Use java 11 for m2e profile as well
Co-authored-by: Christopher Tubbs
---
pom.xml | 21 +
1 file changed, 5 insertions(+), 16 deletions(-)
diff --git a/pom.xml b/pom.xml
index 54064a1..2ca8759 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,7 +20,7 @@
org.apache
apache
-21
+23
org.apache.accumulo
accumulo-testing
@@ -31,8 +31,9 @@
2.1.0-SNAPSHOT
${project.basedir}/contrib/Eclipse-Accumulo-Codestyle.xml
3.2.0
-1.8
-1.8
+11
+11
+11
1.7.25
3.1.7
3.4.14
@@ -158,7 +159,7 @@
net.revelc.code.formatter
formatter-maven-plugin
- 2.10.0
+ 2.14.0
${eclipseFormatterStyle}
${maven.compiler.source}
@@ -388,9 +389,6 @@
m2e.version
-
-8
-
@@ -423,14 +421,5 @@
-
- jdk-release-flag
-
-[9,)
-
-
-8
-
-
[accumulo] branch main updated (7957d67 -> 183387a)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 7957d67 Set reuseForks to true for unit tests, false for ITs (#2007) add 183387a Drop duplicate constructor in ServiceLock (#2008) No new revisions were added by this update. Summary of changes: .../apache/accumulo/fate/zookeeper/ServiceLock.java | 19 +-- .../apache/accumulo/gc/SimpleGarbageCollector.java| 2 +- .../java/org/apache/accumulo/manager/Manager.java | 4 ++-- .../java/org/apache/accumulo/monitor/Monitor.java | 2 +- .../org/apache/accumulo/tserver/TabletServer.java | 3 +-- .../accumulo/test/fate/zookeeper/ServiceLockIT.java | 13 - .../accumulo/test/functional/SplitRecoveryIT.java | 2 +- .../accumulo/test/functional/ZombieTServer.java | 3 +-- 8 files changed, 12 insertions(+), 36 deletions(-)
[accumulo-examples] branch main updated: Add missing job information for MR. Fixes #43 (#73)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new aed79ea Add missing job information for MR. Fixes #43 (#73)
aed79ea is described below
commit aed79ea9edb739c90805202314fadb3455a644e2
Author: Mike Miller
AuthorDate: Fri Apr 2 08:07:21 2021 -0400
Add missing job information for MR. Fixes #43 (#73)
* CharacterHistogram was missing information in the map reduce job so
set the data in the job using legacy methods. This fixes the filedata
example
---
.../accumulo/examples/filedata/CharacterHistogram.java | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git
a/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
b/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
index 0a8aea5..5928786 100644
---
a/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
+++
b/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
@@ -21,7 +21,10 @@ import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map.Entry;
+import java.util.Properties;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -93,9 +96,17 @@ public class CharacterHistogram {
job.setNumReduceTasks(0);
+Properties props = opts.getClientProperties();
+ChunkInputFormat.setZooKeeperInstance(job,
props.getProperty("instance.name"),
+props.getProperty("instance.zookeepers"));
+PasswordToken token = new PasswordToken(props.getProperty("auth.token"));
+ChunkInputFormat.setConnectorInfo(job,
props.getProperty("auth.principal"), token);
+ChunkInputFormat.setInputTableName(job, opts.tableName);
+ChunkInputFormat.setScanAuthorizations(job, opts.auths);
+
job.setOutputFormatClass(AccumuloOutputFormat.class);
AccumuloOutputFormat.configure().clientProperties(opts.getClientProperties())
-.defaultTable(opts.tableName).createTables(true);
+.defaultTable(opts.tableName).createTables(true).store(job);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
[accumulo-examples] branch main updated: Make runmr script use HADOOP_HOME (#72)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 7aa5a4d Make runmr script use HADOOP_HOME (#72)
7aa5a4d is described below
commit 7aa5a4d4cf69a7b534a2833c6a50017a0ad5542a
Author: Mike Miller
AuthorDate: Wed Mar 31 17:20:00 2021 -0400
Make runmr script use HADOOP_HOME (#72)
---
bin/runmr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/runmr b/bin/runmr
index 9de164e..ee65fac 100755
--- a/bin/runmr
+++ b/bin/runmr
@@ -41,4 +41,4 @@ fi
"$ex_home"/bin/build
export HADOOP_USE_CLIENT_CLASSLOADER=true
-yarn jar $EXAMPLES_JAR_PATH "org.apache.accumulo.examples.$1" ${*:2}
+"$HADOOP_HOME"/bin/yarn jar $EXAMPLES_JAR_PATH
"org.apache.accumulo.examples.$1" ${*:2}
[accumulo-examples] branch main updated: Drop TraceDumpExample. Closes #35 (#71)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 10e6e1a Drop TraceDumpExample. Closes #35 (#71)
10e6e1a is described below
commit 10e6e1ad59ff93ca6f7254d44e20268ee476da2c
Author: Mike Miller
AuthorDate: Tue Mar 30 15:00:26 2021 -0400
Drop TraceDumpExample. Closes #35 (#71)
---
contrib/import-control.xml | 8 +-
.../accumulo/examples/client/TraceDumpExample.java | 97 --
2 files changed, 1 insertion(+), 104 deletions(-)
diff --git a/contrib/import-control.xml b/contrib/import-control.xml
index f4628ff..e8b98dc 100644
--- a/contrib/import-control.xml
+++ b/contrib/import-control.xml
@@ -29,15 +29,9 @@
-
-
+
-
-
-
-
-
diff --git
a/src/main/java/org/apache/accumulo/examples/client/TraceDumpExample.java
b/src/main/java/org/apache/accumulo/examples/client/TraceDumpExample.java
deleted file mode 100644
index 36bff49..000
--- a/src/main/java/org/apache/accumulo/examples/client/TraceDumpExample.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.client;
-
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.conf.ClientProperty;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.examples.cli.ClientOnDefaultTable;
-import org.apache.accumulo.examples.cli.ScannerOpts;
-import org.apache.accumulo.tracer.TraceDump;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Example of using the TraceDump class to print a formatted view of a Trace
- */
-public class TraceDumpExample {
- private static final Logger log =
LoggerFactory.getLogger(TraceDumpExample.class);
-
- static class Opts extends ClientOnDefaultTable {
-public Opts() {
- super("trace");
-}
-
-@Parameter(names = {"--traceid"},
-description = "The hex string id of a given trace, for example
16cfbbd7beec4ae3")
-public String traceId = "";
- }
-
- public void dump(Opts opts)
- throws TableNotFoundException, AccumuloException,
AccumuloSecurityException {
-
-if (opts.traceId.isEmpty()) {
- throw new IllegalArgumentException("--traceid option is required");
-}
-
-try (AccumuloClient client = opts.createAccumuloClient()) {
- final String principal =
ClientProperty.AUTH_PRINCIPAL.getValue(opts.getClientProperties());
- final String table = opts.getTableName();
- if (!client.securityOperations().hasTablePermission(principal, table,
TablePermission.READ)) {
-client.securityOperations().grantTablePermission(principal, table,
TablePermission.READ);
-try {
- Thread.sleep(1000);
-} catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new RuntimeException(e);
-}
-while (!client.securityOperations().hasTablePermission(principal,
table,
-TablePermission.READ)) {
- log.info("{} didn't propagate read permission on {}", principal,
table);
- try {
-Thread.sleep(1000);
- } catch (InterruptedException e) {
-Thread.currentThread().interrupt();
-throw new RuntimeException(e);
- }
-}
- }
- Scanner scanner = client.createScanner(table, opts.auths);
- scanner.setRange(new Range(new Text(opts.traceId)));
- TraceDump.printTrace(scanner, System.out::println);
-}
- }
-
- public static void main(String[] args)
-
[accumulo] branch main updated (b0fe6cf -> 391b399)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from b0fe6cf Cleanup Tests (#1982) add 391b399 Drop unused method in Property.java (#1983) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/core/conf/Property.java| 24 -- 1 file changed, 24 deletions(-)
[accumulo] branch main updated (7de7928 -> b0fe6cf)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 7de7928 Fixes #1976 table prop check in Shell (#1979) add b0fe6cf Cleanup Tests (#1982) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/core/data/KeyExtentTest.java| 8 .../org/apache/accumulo/core/data/MutationTest.java | 3 +++ .../core/file/rfile/MultiThreadedRFileTest.java | 8 .../accumulo/core/file/rfile/RelativeKeyTest.java | 1 + .../core/iterators/system/MultiIteratorTest.java| 9 - .../accumulo/core/iterators/user/CombinerTest.java | 5 - .../core/iterators/user/IndexedDocIteratorTest.java | 1 - .../core/iterators/user/IntersectingIteratorTest.java | 2 -- .../server/master/balancer/GroupBalancerTest.java | 4 .../java/org/apache/accumulo/tracer/TracerTest.java | 17 +++-- .../org/apache/accumulo/tserver/InMemoryMapTest.java| 16 .../tserver/tablet/TabletMutationPrepAttemptTest.java | 4 +++- 12 files changed, 10 insertions(+), 68 deletions(-)
[accumulo] branch main updated: Fixes #1976 table prop check in Shell (#1979)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7de7928 Fixes #1976 table prop check in Shell (#1979)
7de7928 is described below
commit 7de7928274b2dca59095957e62d9356dc4b64034
Author: Mike Miller
AuthorDate: Mon Mar 29 15:34:46 2021 -0400
Fixes #1976 table prop check in Shell (#1979)
---
shell/src/main/java/org/apache/accumulo/shell/Shell.java | 2 +-
.../org/apache/accumulo/start/classloader/vfs/ContextManager.java| 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
index 41d83bc..a3ecb76 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@ -484,7 +484,7 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
// look for either the old property or the new one, but
// if the new one is set, stop looking and let it take precedence
if (entry.getKey().equals(Property.TABLE_CLASSLOADER_CONTEXT.getKey())
- && entry.getValue() != null && !entry.getKey().isEmpty()) {
+ && entry.getValue() != null && !entry.getValue().isEmpty()) {
return entry.getValue();
}
@SuppressWarnings("removal")
diff --git
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
index 8c4db84..c53d815 100644
---
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
+++
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
@@ -189,13 +189,12 @@ class ContextManager {
if (loader == null) {
// oops, context was closed by another thread, try again
ClassLoader loader2 = getClassLoader(contextName);
- log.debug("Returning new classloader {} for context {}",
loader2.getClass().getSimpleName(),
+ log.debug("Returning new classloader {} for context {}",
loader2.getClass().getName(),
contextName);
return loader2;
}
-log.debug("Returning classloader {} for context {}",
loader.getClass().getSimpleName(),
-contextName);
+log.debug("Returning classloader {} for context {}",
loader.getClass().getName(), contextName);
return loader;
}
[accumulo-examples] branch main updated: Add missing htrace dep
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 9f7704d Add missing htrace dep
9f7704d is described below
commit 9f7704df126821a7850a8356cddec0bd9ef16056
Author: Mike Miller
AuthorDate: Thu Mar 25 11:14:40 2021 -0400
Add missing htrace dep
---
pom.xml | 9 -
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index e789216..3601f32 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,6 +31,8 @@
2.1.0-SNAPSHOT
contrib/Eclipse-Accumulo-Codestyle.xml
3.3.0
+4.1.0-incubating
+3.2.0-incubating
2.14.0
11
11
@@ -89,7 +91,12 @@
org.apache.htrace
htrace-core
- 3.1.0-incubating
+ ${htrace.version}
+
+
+ org.apache.htrace
+ htrace-core4
+ ${htrace.hadoop.version}
org.apache.logging.log4j
[accumulo-examples] branch main updated: Add more precise excludes to pom to fix hadoop client
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 09e12a9 Add more precise excludes to pom to fix hadoop client 09e12a9 is described below commit 09e12a9934b9eeb3bbf49a2b3132fb781567e85f Author: Mike Miller AuthorDate: Thu Mar 25 10:16:45 2021 -0400 Add more precise excludes to pom to fix hadoop client --- pom.xml | 12 +++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e2e3fd1..e789216 100644 --- a/pom.xml +++ b/pom.xml @@ -321,7 +321,17 @@ org.apache.logging.log4j:log4j-slf4j-impl:jar: javax.activation:javax.activation-api:jar: org.apache.zookeeper:zookeeper-jute:jar: - org.apache.hadoop:* + org.apache.hadoop:hadoop-annotations:jar: + org.apache.hadoop:hadoop-auth:jar: + org.apache.hadoop:hadoop-mapreduce-client-common:jar: + org.apache.hadoop:hadoop-yarn-api:jar: + org.apache.hadoop:hadoop-yarn-common:jar: + org.apache.hadoop:hadoop-mapreduce-client-core:jar: + org.apache.hadoop:hadoop-common:jar: + org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar: + org.apache.hadoop:hadoop-yarn-client:jar: + org.apache.hadoop:hadoop-hdfs-client:jar: + org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:
[accumulo-examples] branch main updated: Fix shade plugin warnings
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new b6bc9d9 Fix shade plugin warnings b6bc9d9 is described below commit b6bc9d9dabb56b8a5e64435e0d45e1c0f828e36f Author: Mike Miller AuthorDate: Thu Mar 25 09:58:13 2021 -0400 Fix shade plugin warnings --- pom.xml | 7 +++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 438fcf0..e2e3fd1 100644 --- a/pom.xml +++ b/pom.xml @@ -316,12 +316,19 @@ org.apache.accumulo:accumulo-native + org.apache.logging.log4j:log4j-api:jar: + org.apache.logging.log4j:log4j-1.2-api:jar: + org.apache.logging.log4j:log4j-slf4j-impl:jar: + javax.activation:javax.activation-api:jar: + org.apache.zookeeper:zookeeper-jute:jar: + org.apache.hadoop:* *:* +module-info.class META-INF/*.SF META-INF/*.DSA META-INF/*.RSA
[accumulo] branch main updated: Refactor Crypto packages (#1956)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 0781550 Refactor Crypto packages (#1956)
0781550 is described below
commit 0781550076f04d12716650fd64881d8b9d041afa
Author: Mike Miller
AuthorDate: Thu Mar 4 17:43:04 2021 -0500
Refactor Crypto packages (#1956)
* Move AESCryptoService and NoCryptoService into public spi package
* Move CryptoEnvironmentImpl into core.crypto package with other server
crypto stuff
* Make AESCryptoUtils static methods on AESCryptoService
---
.../org/apache/accumulo/core/conf/Property.java| 2 +-
.../CryptoEnvironmentImpl.java | 2 +-
.../accumulo/core/crypto/CryptoServiceFactory.java | 2 +-
.../apache/accumulo/core/crypto/CryptoUtils.java | 1 -
.../accumulo/core/cryptoImpl/AESKeyUtils.java | 95 --
.../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java| 6 +-
.../crypto}/AESCryptoService.java | 81 +++---
.../crypto}/NoCryptoService.java | 7 +-
.../crypto}/NoFileDecrypter.java | 5 +-
.../crypto}/NoFileEncrypter.java | 5 +-
.../accumulo/core/conf/SiteConfigurationTest.java | 4 +-
.../apache/accumulo/core/crypto/CryptoTest.java| 30 ---
.../apache/accumulo/core/file/rfile/RFileTest.java | 2 +-
core/src/test/resources/accumulo2.properties | 2 +-
.../org/apache/accumulo/tserver/log/DfsLogger.java | 4 +-
.../test/functional/WriteAheadLogEncryptedIT.java | 2 +-
17 files changed, 99 insertions(+), 153 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 04015e3..c757648 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -176,7 +176,7 @@ public enum Property {
"Sensitive properties related to on-disk file encryption."),
@Experimental
INSTANCE_CRYPTO_SERVICE("instance.crypto.service",
- "org.apache.accumulo.core.cryptoImpl.NoCryptoService",
PropertyType.CLASSNAME,
+ "org.apache.accumulo.core.spi.crypto.NoCryptoService",
PropertyType.CLASSNAME,
"The class which executes on-disk file encryption. The default does
nothing. To enable "
+ "encryption, replace this classname with an implementation of the"
+ "org.apache.accumulo.core.spi.crypto.CryptoService interface."),
diff --git
a/core/src/main/java/org/apache/accumulo/core/cryptoImpl/CryptoEnvironmentImpl.java
b/core/src/main/java/org/apache/accumulo/core/crypto/CryptoEnvironmentImpl.java
similarity index 96%
rename from
core/src/main/java/org/apache/accumulo/core/cryptoImpl/CryptoEnvironmentImpl.java
rename to
core/src/main/java/org/apache/accumulo/core/crypto/CryptoEnvironmentImpl.java
index 17863be..ba2deae 100644
---
a/core/src/main/java/org/apache/accumulo/core/cryptoImpl/CryptoEnvironmentImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/crypto/CryptoEnvironmentImpl.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.core.cryptoImpl;
+package org.apache.accumulo.core.crypto;
import org.apache.accumulo.core.spi.crypto.CryptoEnvironment;
diff --git
a/core/src/main/java/org/apache/accumulo/core/crypto/CryptoServiceFactory.java
b/core/src/main/java/org/apache/accumulo/core/crypto/CryptoServiceFactory.java
index f3c8737..1415b0d 100644
---
a/core/src/main/java/org/apache/accumulo/core/crypto/CryptoServiceFactory.java
+++
b/core/src/main/java/org/apache/accumulo/core/crypto/CryptoServiceFactory.java
@@ -21,8 +21,8 @@ package org.apache.accumulo.core.crypto;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.cryptoImpl.NoCryptoService;
import org.apache.accumulo.core.spi.crypto.CryptoService;
+import org.apache.accumulo.core.spi.crypto.NoCryptoService;
public class CryptoServiceFactory {
diff --git
a/core/src/main/java/org/apache/accumulo/core/crypto/CryptoUtils.java
b/core/src/main/java/org/apache/accumulo/core/crypto/CryptoUtils.java
index 703a0eb..53fff25 100644
--- a/core/src/main/java/org/apache/accumulo/core/crypto/CryptoUtils.java
+++ b/core/src/main/java/org/apache/accumulo/core/crypto/CryptoUtils.java
@@ -26,7 +26,6 @@ import java.security.NoSuchProviderException;
import java.security.SecureRandom;
import java.util.Objects;
-import org.apache.accumulo.core.cryptoImpl.CryptoEnvironm
[accumulo] branch main updated: Clean up CompactionDirectives default impl (#1950)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3bbc77d Clean up CompactionDirectives default impl (#1950)
3bbc77d is described below
commit 3bbc77de13aa6cf0b849f7b28ec08cb6e6b843e9
Author: Mike Miller
AuthorDate: Thu Mar 4 07:27:58 2021 -0500
Clean up CompactionDirectives default impl (#1950)
* Created CompactionDirectivesBuilder class for builder logic
Co-authored-by: Keith Turner
---
.../core/spi/compaction/CompactionDirectives.java | 17 +++--
...tives.java => CompactionDirectivesBuilder.java} | 38 +-
...rectives.java => CompactionDirectivesImpl.java} | 32
.../spi/compaction/CompactionsDirectiveImpl.java | 85 --
.../spi/compaction/SimpleCompactionDispatcher.java | 8 +-
5 files changed, 48 insertions(+), 132 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
index e3370eb..6991fc6 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
@@ -32,17 +32,24 @@ public interface CompactionDirectives {
CompactionServiceId getService();
/**
+ * Required for CompactionDirectives
+ *
* @since 2.1.0
*/
- public static interface Builder {
-Builder setService(CompactionServiceId service);
+ interface ServiceBuilder {
+Builder toService(CompactionServiceId service);
-Builder setService(String compactionServiceId);
+Builder toService(String compactionServiceId);
+ }
+ /**
+ * @since 2.1.0
+ */
+ interface Builder {
CompactionDirectives build();
}
- public static Builder builder() {
-return CompactionsDirectiveImpl.DEFAULT_BUILDER;
+ static ServiceBuilder builder() {
+return new CompactionDirectivesBuilder();
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectivesBuilder.java
similarity index 55%
copy from
core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
copy to
core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectivesBuilder.java
index e3370eb..c542699 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectivesBuilder.java
@@ -18,31 +18,31 @@
*/
package org.apache.accumulo.core.spi.compaction;
+import java.util.Objects;
+
/**
- * The directions of a {@link CompactionDispatcher}
- *
- * @since 2.1.0
- * @see org.apache.accumulo.core.spi.compaction
+ * This class intentionally package private.
*/
-public interface CompactionDirectives {
-
- /**
- * @return The service where a compaction should run.
- */
- CompactionServiceId getService();
+class CompactionDirectivesBuilder
+implements CompactionDirectives.Builder,
CompactionDirectives.ServiceBuilder {
- /**
- * @since 2.1.0
- */
- public static interface Builder {
-Builder setService(CompactionServiceId service);
+ private CompactionServiceId service;
-Builder setService(String compactionServiceId);
+ @Override
+ public CompactionDirectives.Builder toService(CompactionServiceId service) {
+this.service = Objects.requireNonNull(service, "CompactionServiceId cannot
be null");
+return this;
+ }
-CompactionDirectives build();
+ @Override
+ public CompactionDirectives.Builder toService(String compactionServiceId) {
+this.service = CompactionServiceId.of(compactionServiceId);
+return this;
}
- public static Builder builder() {
-return CompactionsDirectiveImpl.DEFAULT_BUILDER;
+ @Override
+ public CompactionDirectives build() {
+return new CompactionDirectivesImpl(service);
}
+
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectivesImpl.java
similarity index 61%
copy from
core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
copy to
core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectivesImpl.java
index e3370eb..fe2a833 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectives.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionDirectivesImpl.java
@@ -19,30 +19,24 @@
package org.apache.accumulo.core.spi.compaction;
/**
- * The directions of a {@link CompactionDispatcher}
- *
- * @sinc
[accumulo] branch main updated (7e3c67d -> 2a4226b)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 7e3c67d Fix typo in RecoveryManager add 2a4226b Create CryptoUtils getFileDecrypter method (#1951) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/core/crypto/CryptoUtils.java | 15 +++ .../org/apache/accumulo/core/crypto/CryptoTest.java | 19 +++ .../org/apache/accumulo/tserver/log/DfsLogger.java| 5 + 3 files changed, 23 insertions(+), 16 deletions(-)
[accumulo] branch main updated: Fix typo in RecoveryManager
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7e3c67d Fix typo in RecoveryManager
7e3c67d is described below
commit 7e3c67d0c0b831d7d29c69e24d07dd700f24c5e0
Author: Mike Miller
AuthorDate: Thu Feb 25 15:37:53 2021 -0500
Fix typo in RecoveryManager
---
.../main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
index 4406f5a..c849b8d 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java
@@ -112,7 +112,7 @@ public class RecoveryManager {
initiateSort(sortId, source, destination);
}
} catch (FileNotFoundException e) {
-log.debug("Unable to initate log sort for " + source + ": " + e);
+log.debug("Unable to initiate log sort for " + source + ": " + e);
} catch (Exception e) {
log.warn("Failed to initiate log sort " + source, e);
} finally {
[accumulo] branch main updated (8718d02 -> a17397d)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 8718d02 Improve tablet details in Monitor. Closes #1940 (#1943) add a17397d Increase difference between test values. Fixes #1789 (#1948) No new revisions were added by this update. Summary of changes: .../tserver/memory/LargestFirstMemoryManager.java | 2 +- .../memory/LargestFirstMemoryManagerTest.java | 55 -- 2 files changed, 30 insertions(+), 27 deletions(-)
[accumulo] branch main updated (65c451a -> 8718d02)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 65c451a Minor warning cleanup from IDE add 8718d02 Improve tablet details in Monitor. Closes #1940 (#1943) No new revisions were added by this update. Summary of changes: .../src/main/resources/org/apache/accumulo/monitor/templates/server.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] branch main updated (97ac5c4 -> 2b1410a)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 97ac5c4 Update plugin/some dependency versions (#1941) add 2b1410a Create readTablets method in Ample. Closes #1473 (#1945) No new revisions were added by this update. Summary of changes: .../accumulo/core/clientImpl/OfflineIterator.java | 4 +- .../core/clientImpl/TableOperationsImpl.java | 4 +- .../clientImpl/bulk/ConcurrentKeyExtentCache.java | 4 +- .../accumulo/core/metadata/schema/Ample.java | 8 +++ .../accumulo/core/metadata/schema/AmpleImpl.java | 10 ++- .../core/metadata/schema/TabletsMetadata.java | 23 --- .../org/apache/accumulo/core/summary/Gatherer.java | 8 +-- .../java/org/apache/accumulo/core/util/Merge.java | 4 +- .../manager/balancer/BalancerEnvironmentImpl.java | 4 +- .../server/master/balancer/GroupBalancer.java | 4 +- .../accumulo/server/util/ListVolumesUsed.java | 4 +- .../accumulo/server/util/MetadataTableUtil.java| 4 +- .../manager/state/RootTabletStateStoreTest.java| 6 ++ .../apache/accumulo/gc/SimpleGarbageCollector.java | 4 +- .../manager/ManagerClientServiceHandler.java | 5 +- .../manager/tableOps/bulkVer2/LoadFiles.java | 4 +- .../manager/tableOps/bulkVer2/PrepBulkImport.java | 7 ++- .../manager/tableOps/compact/CompactionDriver.java | 5 +- .../shell/commands/ListTabletsCommand.java | 2 +- .../apache/accumulo/test/CompactionExecutorIT.java | 4 +- .../accumulo/test/functional/AccumuloClientIT.java | 1 - .../apache/accumulo/test/functional/BulkNewIT.java | 4 +- .../accumulo/test/functional/MetadataIT.java | 73 ++ 23 files changed, 147 insertions(+), 49 deletions(-)
[accumulo-testing] branch main updated: Fix typo in bin/cingest script
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new fbaf16d Fix typo in bin/cingest script fbaf16d is described below commit fbaf16dc2c065eaab690d8d7abd75750d5d0425c Author: Mike Miller AuthorDate: Tue Feb 23 08:28:57 2021 -0500 Fix typo in bin/cingest script --- bin/cingest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cingest b/bin/cingest index ab44b19..0753608 100755 --- a/bin/cingest +++ b/bin/cingest @@ -31,7 +31,7 @@ Available applications: batchwalk Randomly walks the graph using a batch scanner scan Scans the graph verifyVerifies continous ingest test. Stop ingest before running. -moru Streses Accumulo by reading and writing to the ingest table. +moru Stresses Accumulo by reading and writing to the ingest table. Stop ingest before running. bulk Create RFiles in a Map Reduce job and calls importDirectory if successful EOF
[accumulo-testing] branch main updated: Add back END to MultiTable so it can be used by other modules
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new 17963d6 Add back END to MultiTable so it can be used by other modules 17963d6 is described below commit 17963d619f5418254e5dafd930c1d2210f48e937 Author: Mike Miller AuthorDate: Mon Feb 22 14:40:12 2021 -0500 Add back END to MultiTable so it can be used by other modules --- src/main/resources/randomwalk/modules/MultiTable.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/randomwalk/modules/MultiTable.xml b/src/main/resources/randomwalk/modules/MultiTable.xml index 55b7f0b..6698e4a 100644 --- a/src/main/resources/randomwalk/modules/MultiTable.xml +++ b/src/main/resources/randomwalk/modules/MultiTable.xml @@ -34,6 +34,7 @@ +
[accumulo] branch main updated: Stop recovery if tablet is being deleted (#1934)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1052076 Stop recovery if tablet is being deleted (#1934)
1052076 is described below
commit 10520766da991746df86cb94e4acf94a39d00747
Author: Mike Miller
AuthorDate: Fri Feb 19 09:08:40 2021 -0500
Stop recovery if tablet is being deleted (#1934)
---
.../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 1a90b1c..fc8c6fe 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -340,7 +340,8 @@ public class Tablet {
tabletMemory = new TabletMemory(this);
-if (!logEntries.isEmpty()) {
+// don't bother examining WALs for recovery if Table is being deleted
+if (!logEntries.isEmpty() && !isBeingDeleted()) {
TabletLogger.recovering(extent, logEntries);
final AtomicLong entriesUsedOnTablet = new AtomicLong(0);
// track max time from walog entries without timestamps
[accumulo] branch main updated (e6cc640 -> f18a83d)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from e6cc640 Remove unused param from CompactionEnv (#1936) add f18a83d Add stats to new Compactable classes. Fixes #1930 (#1937) No new revisions were added by this update. Summary of changes: .../accumulo/monitor/rest/tservers/TabletServerResource.java | 2 +- .../org/apache/accumulo/tserver/compactions/Compactable.java | 2 +- .../accumulo/tserver/compactions/CompactionExecutor.java | 5 +++-- .../org/apache/accumulo/tserver/tablet/CompactableImpl.java | 12 ++-- .../org/apache/accumulo/tserver/tablet/CompactableUtils.java | 5 - .../main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 9 + 6 files changed, 28 insertions(+), 7 deletions(-)
[accumulo] branch main updated (287a1bd -> e6cc640)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 287a1bd Stop user flush if table is being deleted (#1931) add e6cc640 Remove unused param from CompactionEnv (#1936) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/tserver/tablet/CompactableImpl.java| 2 +- .../java/org/apache/accumulo/tserver/tablet/CompactableUtils.java | 4 ++-- .../src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java | 6 +++--- .../java/org/apache/accumulo/tserver/tablet/MinorCompactor.java | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-)
[accumulo-testing] branch main updated: Add more splits to RW multitable CreateTable
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 66dcd04 Add more splits to RW multitable CreateTable
66dcd04 is described below
commit 66dcd0430ff0b012a75c256f4bb8a462c2ffe14d
Author: Mike Miller
AuthorDate: Wed Feb 17 09:21:10 2021 -0500
Add more splits to RW multitable CreateTable
---
.../apache/accumulo/testing/randomwalk/multitable/CreateTable.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
index b132662..3ef2d27 100644
---
a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
+++
b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CreateTable.java
@@ -36,13 +36,16 @@ public class CreateTable extends Test {
for (int i = 1; i < 10; i++) {
splits.add(new Text(Integer.toString(i)));
}
+for (String s : "a b c d e f".split(" ")) {
+ splits.add(new Text(s));
+}
}
@Override
public void visit(State state, RandWalkEnv env, Properties props) throws
Exception {
AccumuloClient client = env.getAccumuloClient();
-int nextId = ((Integer) state.get("nextId")).intValue();
+int nextId = (Integer) state.get("nextId");
String tableName = String.format("%s_%d",
state.getString("tableNamePrefix"), nextId);
try {
client.tableOperations().create(tableName);
@@ -61,6 +64,6 @@ public class CreateTable extends Test {
}
nextId++;
-state.set("nextId", Integer.valueOf(nextId));
+state.set("nextId", nextId);
}
}
[accumulo] branch main updated: Stop user flush if table is being deleted (#1931)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 287a1bd Stop user flush if table is being deleted (#1931)
287a1bd is described below
commit 287a1bd4c9729ad3b738db91942f3841e852d523
Author: Mike Miller
AuthorDate: Wed Feb 17 08:32:57 2021 -0500
Stop user flush if table is being deleted (#1931)
---
.../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index f4eb7f9..7605500 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -843,7 +843,8 @@ public class Tablet {
return;
}
-if (isClosing() || isClosed() ||
getTabletMemory().memoryReservedForMinC()) {
+if (isClosing() || isClosed() || isBeingDeleted()
+|| getTabletMemory().memoryReservedForMinC()) {
return;
}
[accumulo] branch main updated: Update ServerBulkImportStatus for new Bulk import. Closes #1510 (#1923)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 970b054 Update ServerBulkImportStatus for new Bulk import. Closes
#1510 (#1923)
970b054 is described below
commit 970b0547cc9ae46fc5f95130b407bb36fc9521a9
Author: Mike Miller
AuthorDate: Wed Feb 17 07:42:38 2021 -0500
Update ServerBulkImportStatus for new Bulk import. Closes #1510 (#1923)
---
.../apache/accumulo/server/util/ServerBulkImportStatus.java | 13 +++--
.../org/apache/accumulo/manager/FateServiceHandler.java | 1 +
.../accumulo/manager/tableOps/bulkVer2/BulkImportMove.java | 2 ++
.../manager/tableOps/bulkVer2/CleanUpBulkImport.java| 3 +++
.../accumulo/manager/tableOps/bulkVer2/LoadFiles.java | 2 ++
.../org/apache/accumulo/monitor/templates/bulkImport.ftl| 2 +-
.../org/apache/accumulo/tserver/ThriftClientHandler.java| 8
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
8 files changed, 25 insertions(+), 8 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java
index 0aadcb8..34d87e1 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ServerBulkImportStatus.java
@@ -37,12 +37,13 @@ public class ServerBulkImportStatus {
public void updateBulkImportStatus(List files, BulkImportState
state) {
for (String file : files) {
- BulkImportStatus initial = new
BulkImportStatus(System.currentTimeMillis(), file, state);
- status.putIfAbsent(file, initial);
- initial = status.get(file);
- if (initial != null) {
-initial.state = state;
- }
+ status.compute(file, (key, currentStatus) -> {
+if (currentStatus == null) {
+ return new BulkImportStatus(System.currentTimeMillis(), file, state);
+}
+currentStatus.state = state;
+return currentStatus;
+ });
}
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
index 0db5622..d1ff6dd 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
@@ -599,6 +599,7 @@ class FateServiceHandler implements FateService.Iface {
if (!canBulkImport)
throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
+manager.updateBulkImportStatus(dir, BulkImportState.INITIAL);
manager.fate.seedTransaction(opid,
new TraceRepo<>(new PrepBulkImport(tableId, dir, setTime)),
autoCleanup);
break;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
index 4788afd..838eaf6 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
@@ -30,6 +30,7 @@ import
org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.manager.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.BulkImportState;
import org.apache.accumulo.fate.FateTxId;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.manager.Manager;
@@ -84,6 +85,7 @@ class BulkImportMove extends ManagerRepo {
}
try {
+ manager.updateBulkImportStatus(sourceDir.toString(),
BulkImportState.MOVING);
Map oldToNewNameMap =
BulkSerialize.readRenameMap(bulkDir.toString(), fs::open);
moveFiles(tid, sourceDir, bulkDir, manager, fs, oldToNewNameMap);
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
index 0b3d3cb..d7b0088 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.manager.state.tables.T
[accumulo] 01/02: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3a1d7404b2dc8d910c8a340c19a5326ae2e3cd23 Merge: 0d5ea4b 4b3a3b9 Author: Mike Miller AuthorDate: Tue Feb 16 15:13:04 2021 -0500 Merge remote-tracking branch 'upstream/1.10' into main
[accumulo] 02/02: Improvements to TabletStateChangeIteratorIT. Closes #1932
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 925d17bdd72cf1bbdd0d693d31e58b02fb7d6eab
Author: Mike Miller
AuthorDate: Tue Feb 16 10:37:37 2021 -0500
Improvements to TabletStateChangeIteratorIT. Closes #1932
* Modified copyTable method to gather info about the metadata first
before creating the copy, preventing information about the copy itself
from being seen.
* Added debugging information
* Changed createTable method to pre split tables
---
.../functional/TabletStateChangeIteratorIT.java| 62 ++
1 file changed, 51 insertions(+), 11 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 0a66f3d..f7597fa 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -20,10 +20,12 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
@@ -43,6 +45,7 @@ import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.ClientInfo;
import org.apache.accumulo.core.clientImpl.Tables;
@@ -68,6 +71,8 @@ import
org.apache.accumulo.server.manager.state.MetaDataTableScanner;
import org.apache.accumulo.server.manager.state.TabletStateChangeIterator;
import org.apache.hadoop.io.Text;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
@@ -76,6 +81,7 @@ import com.google.common.collect.Sets;
* in the metadata table when there is no work to be done on the tablet (see
ACCUMULO-3580)
*/
public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
+ private final static Logger log =
LoggerFactory.getLogger(TabletStateChangeIteratorIT.class);
@Override
public int defaultTimeoutSeconds() {
@@ -105,9 +111,12 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
copyTable(client, MetadataTable.NAME, metaCopy1);
State state = new State(client);
- while (findTabletsNeedingAttention(client, metaCopy1, state) > 0) {
+ int tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1,
state);
+ while (tabletsInFlux > 0) {
+log.debug("Waiting for {} tablets for {}", tabletsInFlux, metaCopy1);
UtilWaitThread.sleep(500);
copyTable(client, MetadataTable.NAME, metaCopy1);
+tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, state);
}
assertEquals("No tables should need attention", 0,
findTabletsNeedingAttention(client, metaCopy1, state));
@@ -196,31 +205,39 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
private int findTabletsNeedingAttention(AccumuloClient client, String table,
State state)
throws TableNotFoundException {
int results = 0;
+List resultList = new ArrayList<>();
try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
MetaDataTableScanner.configureScanner(scanner, state);
+ log.debug("Current state = {}", state);
scanner.updateScanIteratorOption("tabletChange", "debug", "1");
for (Entry e : scanner) {
-if (e != null)
+if (e != null) {
results++;
+ resultList.add(e.getKey());
+}
}
}
-
+log.debug("Tablets in flux: {}", resultList);
return results;
}
private void createTable(AccumuloClient client, String t, boolean online)
throws AccumuloSecurityException, AccumuloException,
TableNotFoundException,
TableExistsException {
-client.tableOperations().create(t);
-client.tableOperations().online(t, true);
SortedSet partitionKeys = new TreeSet<>();
partitionKeys.add(new Text("some split"));
-client.tableOperations().addSplits(t, partitionKeys);
+NewTableConfiguration ntc = new
NewTableConfiguration().withSplits(partitionKeys);
+client.tableOperations().create(t, ntc);
+client.tabl
[accumulo] branch main updated (0d5ea4b -> 925d17b)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 0d5ea4b Prevent deleted tablets from being flushed (#1899) add 4b3a3b9 Improvements to TabletStateChangeIteratorIT (#1933) new 3a1d740 Merge remote-tracking branch 'upstream/1.10' into main new 925d17b Improvements to TabletStateChangeIteratorIT. Closes #1932 The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../functional/TabletStateChangeIteratorIT.java| 62 ++ 1 file changed, 51 insertions(+), 11 deletions(-)
[accumulo] branch 1.10 updated: Improvements to TabletStateChangeIteratorIT (#1933)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new 4b3a3b9 Improvements to TabletStateChangeIteratorIT (#1933)
4b3a3b9 is described below
commit 4b3a3b98d8916dc49f0ce54f6b8dca7b9473dc29
Author: Mike Miller
AuthorDate: Tue Feb 16 15:11:26 2021 -0500
Improvements to TabletStateChangeIteratorIT (#1933)
* Modified copyTable method to gather info about the metadata first
before creating the copy, preventing information about the copy itself
from being seen.
* Added debugging information
---
.../functional/TabletStateChangeIteratorIT.java| 53 +++---
1 file changed, 46 insertions(+), 7 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 84c503d..b167ded 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -19,10 +19,12 @@ package org.apache.accumulo.test.functional;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
@@ -63,6 +65,8 @@ import
org.apache.accumulo.server.master.state.TabletStateChangeIterator;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
@@ -71,6 +75,7 @@ import com.google.common.collect.Sets;
* in the metadata table when there is no work to be done on the tablet (see
ACCUMULO-3580)
*/
public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
+ private final static Logger log =
LoggerFactory.getLogger(TabletStateChangeIteratorIT.class);
@Override
public int defaultTimeoutSeconds() {
@@ -97,9 +102,12 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
copyTable(MetadataTable.NAME, metaCopy1);
State state = new State();
-while (findTabletsNeedingAttention(metaCopy1, state) > 0) {
+int tabletsInFlux = findTabletsNeedingAttention(metaCopy1, state);
+while (tabletsInFlux > 0) {
+ log.debug("Waiting for {} tablets for {}", tabletsInFlux, metaCopy1);
UtilWaitThread.sleep(500);
copyTable(MetadataTable.NAME, metaCopy1);
+ tabletsInFlux = findTabletsNeedingAttention(metaCopy1, state);
}
assertEquals("No tables should need attention", 0,
findTabletsNeedingAttention(metaCopy1, state));
@@ -184,14 +192,19 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
private int findTabletsNeedingAttention(String table, State state) throws
TableNotFoundException {
int results = 0;
+List resultList = new ArrayList<>();
+
Scanner scanner = getConnector().createScanner(table,
Authorizations.EMPTY);
MetaDataTableScanner.configureScanner(scanner, state);
+log.debug("Current state = {}", state);
scanner.updateScanIteratorOption("tabletChange", "debug", "1");
for (Entry e : scanner) {
- if (e != null)
+ if (e != null) {
results++;
+resultList.add(e.getKey());
+ }
}
-
+log.debug("Tablets in flux: {}", resultList);
return results;
}
@@ -208,6 +221,10 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
}
}
+ /**
+ * Create a copy of the source table by first gathering all the rows of the
source in a list of
+ * mutations. Then create the copy of the table and apply the mutations to
the copy.
+ */
private void copyTable(String source, String copy) throws AccumuloException,
AccumuloSecurityException, TableNotFoundException, TableExistsException {
try {
@@ -216,10 +233,10 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
// ignored
}
-getConnector().tableOperations().create(copy);
+log.info("Gathering rows to copy {} ", source);
+List mutations = new ArrayList<>();
-try (Scanner scanner = getConnector().createScanner(source,
Authorizations.EMPTY);
-BatchWriter writer = getConnector().createBatchWriter(copy, new
BatchWriterConfig())) {
+try (Scanner scanner = getConnector().createScanner(source,
Authorizations.EMPTY)) {
RowIterator rows = new RowIte
[accumulo] branch main updated: Prevent deleted tablets from being flushed (#1899)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 0d5ea4b Prevent deleted tablets from being flushed (#1899)
0d5ea4b is described below
commit 0d5ea4b51e5f781c35280789d281817d4bb730ee
Author: Mike Miller
AuthorDate: Fri Feb 12 12:13:46 2021 -0500
Prevent deleted tablets from being flushed (#1899)
* Add check to LargestFirstMemoryManager.tabletsToMinorCompact() to
not pick tablets from a table being deleted
* Remove deleted tablet from memory reports in TabletServerResourceManager
so it won't keep trying to flush delete tablets when they are large
* Created isBeingDeleted() in Tablet for checking
* The CleanUp step of deletes will wait until all tablets of a tablet
are unassigned. This will stop the memory mgr from flushing if the table
is being deleted, allowing it to be unassigned faster.
* Added debug to Tablet.completeClose() for better insight when waiting
* Updated LargestFirstMemoryManagerTest to test tablets being deleted
* Added 1 min timeout to LargestFirstMemoryManagerTest in hopes of
helping with timing issues
---
.../tserver/TabletServerResourceManager.java | 4 +--
.../tserver/memory/LargestFirstMemoryManager.java | 17 ++
.../org/apache/accumulo/tserver/tablet/Tablet.java | 13 +++-
.../memory/LargestFirstMemoryManagerTest.java | 38 +-
4 files changed, 54 insertions(+), 18 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index f011ea5..300275e 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -557,13 +557,13 @@ public class TabletServerResourceManager {
}
Tablet tablet = tabletReport.getTablet();
if
(!tablet.initiateMinorCompaction(MinorCompactionReason.SYSTEM)) {
-if (tablet.isClosed()) {
+if (tablet.isClosed() || tablet.isBeingDeleted()) {
// attempt to remove it from the current reports if still
there
synchronized (tabletReports) {
TabletMemoryReport latestReport =
tabletReports.remove(keyExtent);
if (latestReport != null) {
if (latestReport.getTablet() == tablet) {
-log.debug("Cleaned up report for closed tablet {}",
keyExtent);
+log.debug("Cleaned up report for closed/deleted tablet
{}", keyExtent);
} else {
// different tablet instance => put it back
tabletReports.put(keyExtent, latestReport);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
index 7c242b0..c10467c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
@@ -28,6 +28,7 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.server.ServerContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -147,6 +148,10 @@ public class LargestFirstMemoryManager {
return context.getTableConfiguration(tableId) != null;
}
+ protected boolean tableBeingDeleted(TableId tableId) {
+return context.getTableManager().getTableState(tableId) ==
TableState.DELETING;
+ }
+
public List tabletsToMinorCompact(List
tablets) {
if (maxMemory < 0)
throw new IllegalStateException(
@@ -167,9 +172,10 @@ public class LargestFirstMemoryManager {
// find the largest and most idle tablets
for (TabletMemoryReport ts : tablets) {
+ KeyExtent tablet = ts.getExtent();
// Make sure that the table still exists
- if (!tableExists(ts.getExtent().tableId())) {
-log.trace("Ignoring extent for deleted table: {}", ts.getExtent());
+ if (!tableExists(tablet.tableId()) ||
tableBeingDeleted(tablet.tableId())) {
+log.trace("Ignoring extent for deleted table: {}", tablet);
continue;
}
@@ -179,17 +185,16 @@ public class LargestFirstMemoryManager {
[accumulo] branch main updated (f2ecaf6 -> 1a5b46e)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from f2ecaf6 Document master/manager rename in Upgrader9to10 re #1922 (#1927) add 1a5b46e Improve log message in Delete CleanUp FATE. (#1829) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 2ff2618b1c49d01a3ded44173b522c5670cfdeac
Merge: c7b1d85 fbdbda9
Author: Mike Miller
AuthorDate: Thu Feb 11 07:42:33 2021 -0500
Merge remote-tracking branch 'upstream/1.10' into main
core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java | 4 ++--
.../main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java | 3 ++-
.../org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java | 4
3 files changed, 8 insertions(+), 3 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
index fbde1a8,7ea9b81..25c5a52
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
@@@ -209,163 -196,17 +209,163 @@@ public class RFile
/**
* This is an intermediate interface in a larger builder pattern. Supports
setting the required
+ * input sources for reading summary data from an RFile.
+ *
+ * @since 2.0.0
+ */
+ public interface SummaryInputArguments {
+/**
+ * Specify RFiles to read from. When multiple inputs are specified the
summary data will be
+ * merged.
+ *
+ * @param inputs
+ * one or more RFiles to read.
+ * @return this
+ */
+SummaryOptions from(RFileSource... inputs);
+
+/**
+ * Specify RFiles to read from. When multiple are specified the summary
data will be merged.
+ *
+ * @param files
+ * one or more RFiles to read.
+ * @return this
+ */
+SummaryFSOptions from(String... files);
+ }
+
+ /**
+ * This is an intermediate interface in a larger builder pattern. Enables
optionally setting a
+ * FileSystem to read RFile summary data from.
+ *
+ * @since 2.0.0
+ */
+ public interface SummaryFSOptions extends SummaryOptions {
+/**
+ * Optionally provide a FileSystem to open RFiles. If not specified, the
FileSystem will be
+ * constructed using configuration on the classpath.
+ *
+ * @param fs
+ * use this FileSystem to open files.
+ * @return this
+ */
+SummaryOptions withFileSystem(FileSystem fs);
+ }
+
+ /**
+ * This is an intermediate interface in a large builder pattern. Allows
setting options for
+ * retrieving summary data.
+ *
+ * @since 2.0.0
+ */
+ public interface SummaryOptions {
+/**
+ * Retrieve summaries with provided tables properties. Properties for a
table can be obtained by
+ * calling {@link TableOperations#getProperties(String)}. Any property
that impacts file
+ * behavior regardless of whether it has the {@link
Property#TABLE_PREFIX} may be accepted and
+ * used. For example, cache and crypto properties could be passed here.
+ *
+ * @param props
+ * iterable over Accumulo table key value properties.
+ * @return this
+ */
+SummaryOptions withTableProperties(Iterable> props);
+
+/**
+ * @see #withTableProperties(Iterable) Any property that impacts file
behavior regardless of
+ * whether it has the {@link Property#TABLE_PREFIX} may be accepted
and used. For example,
+ * cache and crypto properties could be passed here.
+ * @param props
+ * a map instead of an Iterable
+ * @return this
+ */
+SummaryOptions withTableProperties(Map props);
+
+/**
+ * This method allows retrieving a subset of summary data from a file. If
a file has lots of
+ * separate summaries, reading a subset may be faster.
+ *
+ * @param summarySelector
+ * Only read summary data that was generated with configuration
that this predicate
+ * matches.
+ * @return this
+ */
+SummaryOptions selectSummaries(Predicate
summarySelector);
+
+/**
+ * Summary data may possibly be stored at a more granular level than the
entire file. However
+ * there is no guarantee of this. If the data was stored at a more
granular level, then this
+ * will get a subset of the summary data. The subset will very likely be
an inaccurate
+ * approximation.
+ *
+ * @param startRow
+ * A non-null start row. The startRow is used exclusively.
+ * @return this
+ *
+ * @see FileStatistics#getExtra()
+ */
+SummaryOptions startRow(Text startRow);
+
+/**
+ * @param startRow
+ * UTF-8 encodes startRow. The startRow is used exclusively.
+ * @return this
+ * @see #startRow(Text)
+ */
+SummaryOptions startRow(CharSequence startRow);
+
+/**
+ * Summary data may possibly be stored at a more granular level than the
entire file. However
+ * there is no guaran
[accumulo] branch main updated (c7b1d85 -> 2ff2618)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from c7b1d85 Fix warnings (unused imports, deprecation) add fbdbda9 Improve filename extension usage in RFile API (#1917) new 2ff2618 Merge remote-tracking branch 'upstream/1.10' into main The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java | 4 ++-- .../main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java | 3 ++- .../org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java | 4 3 files changed, 8 insertions(+), 3 deletions(-)
[accumulo] branch 1.10 updated: Improve filename extension usage in RFile API (#1917)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new fbdbda9 Improve filename extension usage in RFile API (#1917)
fbdbda9 is described below
commit fbdbda9c666a989ea5a5b8270f0ca02d4ec30e58
Author: Mike Miller
AuthorDate: Thu Feb 11 07:32:34 2021 -0500
Improve filename extension usage in RFile API (#1917)
* Add info about extension to RFile API javadoc
* Throw IllegalArgumentException early when a filename that does not end
in the appropriate RFile extension is used in the RFileWriterBuilder.
Co-authored-by: Christopher Tubbs
---
core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java | 4 ++--
.../main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java | 3 ++-
.../org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java | 4
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
index fed3e35..7ea9b81 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
@@ -196,14 +196,14 @@ public class RFile {
/**
* This is an intermediate interface in a larger builder pattern. Supports
setting the required
- * output sink to write a RFile to.
+ * output sink to write a RFile to. The filename parameter requires the
".rf" extension.
*
* @since 1.8.0
*/
public static interface OutputArguments {
/**
* @param filename
- * name of file to write RFile data
+ * name of file to write RFile data, ending with the ".rf"
extension
* @return this
*/
public WriterFSOptions to(String filename);
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
index a0703fd..dc02504 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
@@ -47,6 +47,7 @@ import com.google.common.base.Preconditions;
* Keys must be appended in sorted order within a locality group.
* Locality groups must have a mutually exclusive set of column
families.
* The default locality group must be started last.
+ * If using RFile.newWriter().to("filename.rf"), the ".rf" extension is
required.
*
*
*
@@ -58,7 +59,7 @@ import com.google.common.base.Preconditions;
* IterableEntryKey, Value localityGroup2Data = ...
* IterableEntryKey, Value defaultGroupData = ...
*
- * try(RFileWriter writer = RFile.newWriter().to(file).build()) {
+ * try(RFileWriter writer = RFile.newWriter().to("filename.rf").build()) {
*
* // Start a locality group before appending data.
* writer.startNewLocalityGroup("groupA", "columnFam1", "columnFam2");
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
index 4dd7184..d8476eb 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
@@ -111,6 +111,10 @@ class RFileWriterBuilder implements RFile.OutputArguments,
RFile.WriterFSOptions
@Override
public WriterFSOptions to(String filename) {
Objects.requireNonNull(filename);
+if (!filename.endsWith(".rf")) {
+ throw new IllegalArgumentException(
+ "Provided filename (" + filename + ") does not end with '.rf'");
+}
this.out = new OutputArgs(filename);
return this;
}
[accumulo] branch main updated (a75dad4 -> 3edd539)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from a75dad4 fixes #1909 - test broken by changes in commit 954a55395 (#1915) add 3edd539 Drop debug logs to trace in ZooLock (#1918) No new revisions were added by this update. Summary of changes: core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java | 7 ++- 1 file changed, 2 insertions(+), 5 deletions(-)
[accumulo-testing] branch main updated: Add a Bulk import to randomwalk MultiTable (#134)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 689d6d4 Add a Bulk import to randomwalk MultiTable (#134)
689d6d4 is described below
commit 689d6d411e5fda3aeca292656049de1718e949d5
Author: Mike Miller
AuthorDate: Tue Feb 9 16:48:55 2021 -0500
Add a Bulk import to randomwalk MultiTable (#134)
* Create new BulkImport test in MultiTable for more realistic case
* Tweak MultiTable.xml to drop 20 tables, same as create. This will keep
the number of talbes down when running for long period.
* Also set log4j testing logging to DEBUG
---
conf/log4j.properties | 2 +-
.../testing/randomwalk/multitable/BulkImport.java | 119 +
.../randomwalk/multitable/MultiTableFixture.java | 3 +
.../resources/randomwalk/modules/MultiTable.xml| 10 +-
4 files changed, 130 insertions(+), 4 deletions(-)
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 726fb2c..525a6e4 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -19,7 +19,7 @@ log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c{3}] %-5p: %m%n
log4j.logger.org.apache.accumulo=WARN
-log4j.logger.org.apache.accumulo.testing=INFO
+log4j.logger.org.apache.accumulo.testing=DEBUG
log4j.logger.org.apache.curator=ERROR
log4j.logger.org.apache.hadoop=WARN
log4j.logger.org.apache.hadoop.mapreduce=ERROR
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
new file mode 100644
index 000..4c43b0a
--- /dev/null
+++
b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.randomwalk.multitable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.client.IteratorSetting.Column;
+import org.apache.accumulo.core.client.rfile.RFile;
+import org.apache.accumulo.core.client.rfile.RFileWriter;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
+import org.apache.accumulo.testing.randomwalk.State;
+import org.apache.accumulo.testing.randomwalk.Test;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+public class BulkImport extends Test {
+
+ public static final int LOTS = 10;
+ public static final int COLS = 10;
+ public static final List COLNAMES = new ArrayList<>();
+ public static final Text CHECK_COLUMN_FAMILY = new Text("cf");
+ static {
+for (int i = 0; i < COLS; i++) {
+ COLNAMES.add(new Column(CHECK_COLUMN_FAMILY, new
Text(String.format("%03d", i;
+}
+ }
+ public static final Text MARKER_CF = new Text("marker");
+ static final AtomicLong counter = new AtomicLong();
+
+ private static final Value ONE = new Value("1".getBytes());
+
+ /**
+ * Tests both the legacy (deprecated) and new bulk import methods.
+ */
+ @SuppressWarnings({"deprecation", "unchecked"})
+ public void visit(final State state, final RandWalkEnv env, Properties
props) throws Exception {
+List tables = (List) state.get("tableList");
+
+if (tables.isEmpty()) {
+ log.debug("No tables to ingest into");
+ return;
+}
+
+Random rand = new Random();
+String tableName = tables.get(rand.nextInt(tables.size()));
+
+String uuid = UUID.randomUUID().toString();
+final Path dir = new Path("/tmp/bulk", uuid);
+final Path fail = new Path(dir.toS
[accumulo-website] branch asf-site updated (495569e -> 0db6439)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from 495569e Automatic Site Publish by Buildbot add 1ecd555 Automatic Site Publish by Buildbot add 0db6439 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/docs/2.x/administration/caching.html| 4 ++- output/docs/2.x/administration/erasure-coding.html | 4 ++- output/docs/2.x/administration/fate.html | 4 ++- .../docs/2.x/administration/in-depth-install.html | 4 ++- .../2.x/administration/monitoring-metrics.html | 4 ++- output/docs/2.x/administration/multivolume.html| 4 ++- output/docs/2.x/administration/replication.html| 4 ++- output/docs/2.x/administration/scan-executors.html | 4 ++- output/docs/2.x/administration/upgrading.html | 4 ++- .../docs/2.x/configuration/client-properties.html | 4 ++- output/docs/2.x/configuration/files.html | 4 ++- output/docs/2.x/configuration/overview.html| 4 ++- .../docs/2.x/configuration/server-properties.html | 4 ++- output/docs/2.x/development/development_tools.html | 4 ++- output/docs/2.x/development/high_speed_ingest.html | 4 ++- output/docs/2.x/development/iterators.html | 4 ++- output/docs/2.x/development/mapreduce.html | 4 ++- output/docs/2.x/development/proxy.html | 4 ++- output/docs/2.x/development/sampling.html | 4 ++- output/docs/2.x/development/spark.html | 4 ++- output/docs/2.x/development/summaries.html | 4 ++- output/docs/2.x/getting-started/clients.html | 4 ++- output/docs/2.x/getting-started/design.html| 4 ++- output/docs/2.x/getting-started/features.html | 4 ++- output/docs/2.x/getting-started/glossary.html | 4 ++- output/docs/2.x/getting-started/quickstart.html| 28 +--- output/docs/2.x/getting-started/shell.html | 4 ++- .../2.x/getting-started/table_configuration.html | 4 ++- output/docs/2.x/getting-started/table_design.html | 4 ++- output/docs/2.x/security/authentication.html | 4 ++- output/docs/2.x/security/authorizations.html | 4 ++- output/docs/2.x/security/kerberos.html | 4 ++- output/docs/2.x/security/on-disk-encryption.html | 4 ++- output/docs/2.x/security/overview.html | 4 ++- output/docs/2.x/security/permissions.html | 4 ++- output/docs/2.x/security/wire-encryption.html | 4 ++- output/docs/2.x/troubleshooting/advanced.html | 4 ++- output/docs/2.x/troubleshooting/basic.html | 4 ++- output/docs/2.x/troubleshooting/performance.html | 4 ++- .../troubleshooting/system-metadata-tables.html| 4 ++- output/docs/2.x/troubleshooting/tools.html | 4 ++- output/docs/2.x/troubleshooting/tracing.html | 4 ++- output/feed.xml| 4 +-- output/release/accumulo-2.1.0/index.html | 37 +- output/search_data.json| 6 ++-- 45 files changed, 180 insertions(+), 59 deletions(-)
[accumulo-website] branch main updated: Make 2.x manual landing page more clear (#262)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new ca97fed Make 2.x manual landing page more clear (#262)
ca97fed is described below
commit ca97fedba96f631611789bf3378754b152d73d6a
Author: Mike Miller
AuthorDate: Mon Feb 8 13:45:17 2021 -0500
Make 2.x manual landing page more clear (#262)
* Modify quickstart.md to have a title for the whole 2.x manual
* Add skip_doc_h1 tag to drop the title as the first header so it
can be different than the navigation link
---
_docs-2/getting-started/quickstart.md | 20 ++--
_layouts/docs-2.html | 2 ++
2 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/_docs-2/getting-started/quickstart.md
b/_docs-2/getting-started/quickstart.md
index db324ab..0fd44db 100644
--- a/_docs-2/getting-started/quickstart.md
+++ b/_docs-2/getting-started/quickstart.md
@@ -1,22 +1,29 @@
---
-title: Quick Start
+title: Setup
category: getting-started
order: 1
+skip_doc_h1: true
---
+# User Manual (2.x)
-This quick start provides basic instructions for installing and running
Accumulo. For detailed instructions,
-see the [in-depth installation guide][in-depth].
+Starting with Accumulo 2.0, the user manual now lives on the website as a
series of web pages. Previously, it was one large
+pdf document that was only generated during a release. The user manual can now
be updated very quickly and indexed
+for searching across many webpages.
-## Consider using automated tools
+The manual can now be
+searched using the [Search link][search] at the top of the website or
navigated by clicking the links to the left. If you are new
+to Accumulo, follow the instructions below to get started. For detailed
instructions, see the [in-depth installation guide][in-depth].
+
+## Setup for testing or development
If you are setting up Accumulo for **testing or development,** consider using
the following tools:
* [Uno] sets up Accumulo on a single machine for development
* [Muchos] sets up Accumulo on a cluster (optionally launched in Amazon EC2
and Microsoft Azure VM)
-If you are setting up Accumulo for a **production** environment, follow the
instructions in this quick start.
+If you are setting up Accumulo for a **production** environment, follow the
instructions below.
-## Install Accumulo
+## Setup for Production
Either [download] or [build] a binary distribution of Accumulo from source
code and
unpack as follows.
@@ -200,6 +207,7 @@ When finished, use the following commands to stop Accumulo:
* Stop Accumulo service: `accumulo-service tserver stop`
* Stop Accumulo cluster: `accumulo-cluster stop`
+[search]: {{ site.baseurl }}/search
[in-depth]: {% durl administration/in-depth-install %}
[download]: {{ site.baseurl }}/downloads
[build]: https://github.com/apache/accumulo/blob/main/README.md#building
diff --git a/_layouts/docs-2.html b/_layouts/docs-2.html
index 5c10475..262d568 100644
--- a/_layouts/docs-2.html
+++ b/_layouts/docs-2.html
@@ -38,10 +38,12 @@ skiph1fortitle: true
Accumulo {{ page.version }} Documentation {{
page.category | capitalize | replace: "-", " " }} {{
page.title }}
{% endif %}
+{% unless page.skip_doc_h1 %}
{{ page.title }}
https://github.com/apache/accumulo-website/edit/main/{{ page.path }}"
role="button"> Edit this
page
+{% endunless %}
{{ content }}
[accumulo] branch main updated (c00fb38 -> 2bc2415)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from c00fb38 Fix log file names in minicluster and two broken ITs. (#1914) add daa6985 Add a Bulk import to randomwalk MultiTable (#1911) add 2bc2415 Merge remote-tracking branch 'upstream/1.10' into main No new revisions were added by this update. Summary of changes:
[accumulo] branch 1.10 updated: Add a Bulk import to randomwalk MultiTable (#1911)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new daa6985 Add a Bulk import to randomwalk MultiTable (#1911)
daa6985 is described below
commit daa6985409618e6242cbaf693a908d94abd55d9e
Author: Mike Miller
AuthorDate: Mon Feb 8 12:24:15 2021 -0500
Add a Bulk import to randomwalk MultiTable (#1911)
* Create new BulkImport test in MultiTable for more realistic case
* Other improvements to Randomwalk MultiTable
* Modify randomwalk.conf.example to work with Uno by default
---
.../test/randomwalk/multitable/BulkImport.java | 115 +
.../randomwalk/multitable/MultiTableFixture.java | 3 +
.../test/randomwalk/multitable/OfflineTable.java | 10 +-
test/system/randomwalk/conf/modules/MultiTable.xml | 8 +-
.../system/randomwalk/conf/randomwalk.conf.example | 2 +-
5 files changed, 131 insertions(+), 7 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/BulkImport.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/BulkImport.java
new file mode 100644
index 000..1dc9316
--- /dev/null
+++
b/test/src/main/java/org/apache/accumulo/test/randomwalk/multitable/BulkImport.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.randomwalk.multitable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.client.IteratorSetting.Column;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.test.randomwalk.Environment;
+import org.apache.accumulo.test.randomwalk.State;
+import org.apache.accumulo.test.randomwalk.Test;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+public class BulkImport extends Test {
+
+ public static final int LOTS = 10;
+ public static final int COLS = 10;
+ public static final List COLNAMES = new ArrayList<>();
+ public static final Text CHECK_COLUMN_FAMILY = new Text("cf");
+ static {
+for (int i = 0; i < COLS; i++) {
+ COLNAMES.add(new Column(CHECK_COLUMN_FAMILY, new
Text(String.format("%03d", i;
+}
+ }
+ public static final Text MARKER_CF = new Text("marker");
+ static final AtomicLong counter = new AtomicLong();
+
+ private static final Value ONE = new Value("1".getBytes());
+
+ public void visit(final State state, final Environment env, Properties
props) throws Exception {
+@SuppressWarnings("unchecked")
+List tables = (List) state.get("tableList");
+
+if (tables.isEmpty()) {
+ log.debug("No tables to ingest into");
+ return;
+}
+
+Random rand = new Random();
+String tableName = tables.get(rand.nextInt(tables.size()));
+
+String uuid = UUID.randomUUID().toString();
+final Path dir = new Path("/tmp", "bulk_" + uuid);
+final Path fail = new Path(dir.toString() + "_fail");
+final DefaultConfiguration defaultConfiguration =
+AccumuloConfiguration.getDefaultConfiguration();
+final FileSystem fs = (FileSystem) state.get("fs");
+fs.mkdirs(fail);
+final int parts = rand.nextInt(10) + 1;
+
+TreeSet rows = new TreeSet<>();
+for (int i = 0; i < LOTS; i++)
+ rows.add(uuid + String.format("__%05d", i));
+
+String markerColumnQualifier = String.format("%07d",
counter.incremen
[accumulo] branch main updated (8ed8ff6 -> fd001c9)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 8ed8ff6 Improvements to BulkImportCacheCleaner (#1890) add fd001c9 Revert "Add check to not flush when table is being deleted. (#1887)" No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java| 6 -- 1 file changed, 6 deletions(-)
[accumulo] branch main updated (b8dac78 -> 8ed8ff6)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from b8dac78 Add check to not flush when table is being deleted. (#1887) add 8ed8ff6 Improvements to BulkImportCacheCleaner (#1890) No new revisions were added by this update. Summary of changes: .../apache/accumulo/server/zookeeper/TransactionWatcher.java | 12 +++- .../accumulo/tserver/tablet/BulkImportCacheCleaner.java | 8 ++-- 2 files changed, 13 insertions(+), 7 deletions(-)
[accumulo] branch main updated (6af856b -> b8dac78)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 6af856b Merge remote-tracking branch 'upstream/1.10' into main add b8dac78 Add check to not flush when table is being deleted. (#1887) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java| 6 ++ 1 file changed, 6 insertions(+)
[accumulo] branch main updated (ecddf35 -> 6af856b)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from ecddf35 Merge remote-tracking branch 'upstream/1.10' into main add ed97eac Make UnloadTabletHandler log before waiting (#1881) new 6af856b Merge remote-tracking branch 'upstream/1.10' into main The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java | 1 + 1 file changed, 1 insertion(+)
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6af856b38bb57f49418547a8b9c262c66a1fe31e
Merge: ecddf35 ed97eac
Author: Mike Miller
AuthorDate: Wed Jan 27 08:20:57 2021 -0500
Merge remote-tracking branch 'upstream/1.10' into main
.../src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java | 1 +
1 file changed, 1 insertion(+)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
index e3978f0,000..798d7b4
mode 100644,00..100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
@@@ -1,141 -1,0 +1,142 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletLoadState;
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.TabletLocationState;
+import
org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException;
+import org.apache.accumulo.core.tabletserver.thrift.TUnloadTabletGoal;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
+import org.apache.accumulo.server.master.state.TabletStateStore;
+import org.apache.accumulo.tserver.mastermessage.TabletStatusMessage;
+import org.apache.accumulo.tserver.tablet.Tablet;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class UnloadTabletHandler implements Runnable {
+ private static final Logger log =
LoggerFactory.getLogger(UnloadTabletHandler.class);
+ private final KeyExtent extent;
+ private final TUnloadTabletGoal goalState;
+ private final long requestTimeSkew;
+ private final TabletServer server;
+
+ public UnloadTabletHandler(TabletServer server, KeyExtent extent,
TUnloadTabletGoal goalState,
+ long requestTime) {
+this.extent = extent;
+this.goalState = goalState;
+this.server = server;
+this.requestTimeSkew = requestTime -
MILLISECONDS.convert(System.nanoTime(), NANOSECONDS);
+ }
+
+ @Override
+ public void run() {
+
+Tablet t = null;
+
+synchronized (server.unopenedTablets) {
+ if (server.unopenedTablets.contains(extent)) {
+server.unopenedTablets.remove(extent);
+// enqueueMasterMessage(new TabletUnloadedMessage(extent));
+return;
+ }
+}
+synchronized (server.openingTablets) {
+ while (server.openingTablets.contains(extent)) {
+try {
++ log.info("Waiting for tablet {} to finish opening before
unloading.", extent);
+ server.openingTablets.wait();
+} catch (InterruptedException e) {}
+ }
+}
+synchronized (server.onlineTablets) {
+ if (server.onlineTablets.snapshot().containsKey(extent)) {
+t = server.onlineTablets.snapshot().get(extent);
+ }
+}
+
+if (t == null) {
+ // Tablet has probably been recently unloaded: repeated master
+ // unload request is crossing the successful unloaded message
+ if (!server.recentlyUnloadedCache.containsKey(extent)) {
+log.info("told to unload tablet that was not being served {}",
extent);
+server.enqueueMasterMessage(
+new
TabletStatusMessage(TabletLoadState.UNLOAD_FAILURE_NOT_SERVING, extent));
+ }
+ return;
+}
+
+try {
+ t.close(!goalState.equals(TUnloadTabletGoal.DELETED));
+} catch (Exception e) {
+
+ if ((t.isClosing() || t.isClosed()) && e instanceof
IllegalStateException) {
+log.debug("Failed to unload tablet {}... it was already closing or
closed : {}", extent,
+e.g
[accumulo] branch 1.10 updated: Make UnloadTabletHandler log before waiting (#1881)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new ed97eac Make UnloadTabletHandler log before waiting (#1881)
ed97eac is described below
commit ed97eac76da47f711f8f901810cc9d4d8fb27214
Author: Mike Miller
AuthorDate: Wed Jan 27 08:09:54 2021 -0500
Make UnloadTabletHandler log before waiting (#1881)
* If a tablet is in the process of being opened, the UnloadTabletHandler
will wait for a tablet to be removed from the list before proceeding
with the unload. This could take a long time so log info before waiting.
* Fix Typo in UnloadTabletHandler
---
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 73bc58f..e141de9 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -2220,6 +2220,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
synchronized (openingTablets) {
while (openingTablets.contains(extent)) {
try {
+log.info("Waiting for tablet {} to finish opening before
unloading.", extent);
openingTablets.wait();
} catch (InterruptedException e) {}
}
@@ -2246,7 +2247,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
} catch (Throwable e) {
if ((t.isClosing() || t.isClosed()) && e instanceof
IllegalStateException) {
- log.debug("Failed to unload tablet {} ... it was alread closing or
closed : {}", extent,
+ log.debug("Failed to unload tablet {} ... it was already closing or
closed : {}", extent,
e.getMessage());
} else {
log.error("Failed to close tablet {}... Aborting migration", extent,
e);
[accumulo] branch main updated (300c933 -> ecddf35)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 300c933 Deprecate/replace 'master.*' properties with 'manager.*'. (fixes #1640) (#1873) add 48e483b Add tserver to badservers instead of dropping. Fixes #1775 new ecddf35 Merge remote-tracking branch 'upstream/1.10' into main The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ecddf35a9e84be668caf0837166abc07ff67594a Merge: 300c933 48e483b Author: Mike Miller AuthorDate: Tue Jan 26 09:40:56 2021 -0500 Merge remote-tracking branch 'upstream/1.10' into main * git merge upstream/1.10 -sours
[accumulo] branch 1.10 updated (0187567 -> 48e483b)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch 1.10 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 0187567 Re-throw exception from LoggingRunnable (re #1808) add 48e483b Add tserver to badservers instead of dropping. Fixes #1775 No new revisions were added by this update. Summary of changes: server/master/src/main/java/org/apache/accumulo/master/Master.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] branch main updated (2db6d44 -> 9fdfaff)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 2db6d44 Move tests using curator testing server to more general package. (#1882) add 9fdfaff Add tserver to badservers instead of dropping. Fixes #1775 (#1878) No new revisions were added by this update. Summary of changes: server/manager/src/main/java/org/apache/accumulo/master/Master.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo-website] branch asf-site updated (44f239a -> 495569e)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from 44f239a Automatic Site Publish by Buildbot add 9022d1b Automatic Site Publish by Buildbot add 495569e Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 +-- output/people/index.html | 16 ++--- output/tour/authorizations/index.html | 67 --- 3 files changed, 44 insertions(+), 43 deletions(-)
[accumulo-website] branch main updated: Update Authorizations.md and People.md (#260)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 52301d9 Update Authorizations.md and People.md (#260)
52301d9 is described below
commit 52301d9e789512d575625839f04147764dc40573
Author: slackwinner <50567198+slackwin...@users.noreply.github.com>
AuthorDate: Mon Jan 25 11:25:58 2021 -0500
Update Authorizations.md and People.md (#260)
* Applied a few minor changes in Authorizations.md file to fix formatting
issue on Accumulo website. In addition, added my name to the people.md to
receive credit for #255.
---
pages/people.md| 1 +
tour/authorizations.md | 15 ---
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/pages/people.md b/pages/people.md
index 65af7ad..e61afb6 100644
--- a/pages/people.md
+++ b/pages/people.md
@@ -85,6 +85,7 @@ GitHub also has a [contributor list][github-contributors]
based on commits.
| Christian Rohling | [Endgame][ENDGAME]
| [ET][ET] |
| Craig Scheiderer| [Arctic Slope Regional Corp.][ASRC]
| [ET][ET] |
| Damon Brown | [Tetra Concepts LLC][TETRA]
| [ET][ET] |
+| Dane Magbuhos |
| [ET][ET] |
| Dave Wang | [Cloudera][CLOUDERA]
| [PT][PT] |
| David M. Lyle |
| |
| David Protzman |
| |
diff --git a/tour/authorizations.md b/tour/authorizations.md
index 99383da..f21e7aa 100644
--- a/tour/authorizations.md
+++ b/tour/authorizations.md
@@ -16,7 +16,8 @@ For example:
We now want to secure our secret identities of the heroes so that only users
with the proper authorizations can read their names.
-1. Using the code from the previous exercise, add the following to the
beginning of the _exercise_ method.
+1\. Using the code from the previous exercise, add the following to the
beginning of the _exercise_ method.
+
```java
// Create a "secretId" authorization & visibility
final String secretId = "secretId";
@@ -28,25 +29,25 @@ We now want to secure our secret identities of the heroes
so that only users wit
client.securityOperations().changeUserAuthorizations("commissioner", auths);
client.securityOperations().grantTablePermission("commissioner", "GothamPD",
TablePermission.READ);
```
-
-2. The [Mutation] API allows you to set the `secretId` visibility on a column.
Find the proper method for setting a column visibility in
+2\. The [Mutation] API allows you to set the `secretId` visibility on a
column. Find the proper method for setting a column visibility in
the Mutation API and modify the code so the `colVis` variable created above
secures the "name" columns.
-3. Build and run. What data do you see?
+3\. Build and run. What data do you see?
* You should see all of the data except the secret identities of Batman and
Robin. This is because the `Scanner` was created from the root user which
doesn't have the `secretId` authorization.
* Replace the `Authorizations.EMPTY` in the Scanner with the `auths` variable
created above and run it again.
* This should result in an error since the root user doesn't have the
authorizations we tried to pass to the Scanner.
-4. Use the following to create a client for the "commissioner" using the
[Accumulo] entry point.
+4\. Use the following to create a client for the "commissioner" using the
[Accumulo] entry point.
+
```java
try (AccumuloClient commishClient =
Accumulo.newClient().from(client.properties()).as("commissioner",
"gordonrocks").build()) {
// Insert your code here
}
```
+5\. Using the commissioner client, create a Scanner with the authorizations
needed to view the secret identities.
-5. Using the commissioner client, create a Scanner with the authorizations
needed to view the secret identities.
+6\. Build and run. You should see all the rows in the GothamPD table printed,
including these secured key/value pairs:
-6. Build and run. You should see all the rows in the GothamPD table printed,
including these secured key/value pairs:
```commandline
Key : id0001 hero:name [secretId] 1511900180231 false Value : Bruce
Wayne
Key : id0002 hero:name [secretId] 1511900180231 false Value : Dick
Grayson
[accumulo] branch main updated: Add cast checks to compaction cancel method. Fixes #1875 (#1876)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 06251db Add cast checks to compaction cancel method. Fixes #1875
(#1876)
06251db is described below
commit 06251dba99a8655c0db6786866447a6995976eb2
Author: Mike Miller
AuthorDate: Thu Jan 21 13:22:21 2021 -0500
Add cast checks to compaction cancel method. Fixes #1875 (#1876)
---
.../accumulo/tserver/compactions/CompactionExecutor.java | 14 +-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
index a743509..21cf147 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionExecutor.java
@@ -117,7 +117,19 @@ public class CompactionExecutor {
// Occasionally clean the queue of canceled tasks that have hung
around because of their low
// priority. This runs periodically, instead of every time something
is canceled, to avoid
// hurting performance.
-queue.removeIf(runnable -> ((CompactionTask) runnable).getStatus() ==
Status.CANCELED);
+queue.removeIf(runnable -> {
+ CompactionTask compactionTask;
+ if (runnable instanceof TraceRunnable) {
+runnable = ((TraceRunnable) runnable).getRunnable();
+ }
+ if (runnable instanceof CompactionTask) {
+compactionTask = (CompactionTask) runnable;
+ } else {
+throw new IllegalArgumentException(
+"Unknown runnable type " + runnable.getClass().getName());
+ }
+ return compactionTask.getStatus() == Status.CANCELED;
+});
}
return canceled;
[accumulo] branch main updated (3e23ad8 -> 9f549b2)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 3e23ad8 Fixes #1665 use old compaction thread pool config if set (#1861) add 076f1b3 Add check for finished to LogSorter (#1863) new 9f549b2 Merge remote-tracking branch 'upstream/1.10' into main The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/log/LogSorter.java | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-)
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9f549b2fe19666138290913f5660f8bcc83e9f51
Merge: 3e23ad8 076f1b3
Author: Mike Miller
AuthorDate: Tue Jan 12 14:50:10 2021 -0500
Merge remote-tracking branch 'upstream/1.10' into main
Conflicts:
server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
.../src/main/java/org/apache/accumulo/tserver/log/LogSorter.java | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index a9c47b35,4d43da0..1a8ac2f
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@@ -107,7 -109,13 +106,13 @@@ public class LogSorter
String formerThreadName = Thread.currentThread().getName();
int part = 0;
try {
+ // check for finished first since another thread may have already
done the sort
+ if (fs.exists(SortedLogState.getFinishedMarkerPath(destPath))) {
+ log.debug("Sorting already finished at {}", destPath);
+ return;
+ }
-log.info("Copying " + srcPath + " to " + destPath);
++log.info("Copying {} to {}", srcPath, destPath);
// the following call does not throw an exception if the file/dir
does not exist
fs.deleteRecursively(new Path(destPath));
[accumulo] branch 1.10 updated (e6bba1c -> 076f1b3)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch 1.10 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from e6bba1c Merge pull request #1860 from brianloss/1.10 add 076f1b3 Add check for finished to LogSorter (#1863) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/log/LogSorter.java | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-)
[accumulo-testing] branch main updated: Exclude log4j2 jars from shade plugin (#133)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new fed6712 Exclude log4j2 jars from shade plugin (#133) fed6712 is described below commit fed67123411e29decc5ebe17181e280bc549bd88 Author: Mike Miller AuthorDate: Fri Jan 8 08:15:28 2021 -0500 Exclude log4j2 jars from shade plugin (#133) --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 510bc4e..19f3814 100644 --- a/pom.xml +++ b/pom.xml @@ -356,6 +356,8 @@ org.apache.accumulo:accumulo-native org.apache.hadoop:* + org.apache.logging.log4j:log4j-api:jar: + org.apache.logging.log4j:log4j-1.2-api:jar:
[accumulo] branch main updated (ede736f -> 0aff4d2)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from ede736f Remove unneeded resource warning suppressions (#1852) add 0aff4d2 Get proper value in DefaultCompactionPlanner. Fixes #1853 (#1854) No new revisions were added by this update. Summary of changes: .../apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo-testing] branch main updated: Fix warning in randomwalk Init
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 02a39c5 Fix warning in randomwalk Init
02a39c5 is described below
commit 02a39c518bd40748949d0412bc7698b964d8c910
Author: Mike Miller
AuthorDate: Tue Jan 5 07:48:13 2021 -0500
Fix warning in randomwalk Init
* Fix redundant cast to java.util.Random warning in randomwalk Init
---
.../java/org/apache/accumulo/testing/randomwalk/conditional/Init.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/conditional/Init.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/conditional/Init.java
index 2698809..acc6e98 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/conditional/Init.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/conditional/Init.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.testing.randomwalk.conditional;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Properties;
-import java.util.Random;
import java.util.TreeSet;
import org.apache.accumulo.core.client.ConditionalWriter;
@@ -53,7 +52,7 @@ public class Init extends Test {
for (int i = 0; i < numBanks; i++)
banks.add(i);
// shuffle for case when multiple threads are adding banks
-Collections.shuffle(banks, (Random) state.getRandom());
+Collections.shuffle(banks, state.getRandom());
ConditionalWriter cw = (ConditionalWriter) state.get("cw");
[accumulo] branch main updated: Create listtablets shell command. Closes #1317 (#1821)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 376910e Create listtablets shell command. Closes #1317 (#1821)
376910e is described below
commit 376910eb26e51d7361f3852705a954ad998e4c6b
Author: Mike Miller
AuthorDate: Mon Jan 4 15:16:08 2021 -0500
Create listtablets shell command. Closes #1317 (#1821)
* New command for debugging tablets called listtablets
* Added getLiveTServers() to TabletMetadata for generating a list of
tservers that currently have a lock in ZK, similar to master.
* The list of live tservers is passed to TabletMetadata in order to get
the current state of a tablet
* Command will print one line for every tablet in a table
* Created TabletMetadataIT for testing getLiveTServers()
Co-authored-by: EdColeman
---
.../core/metadata/schema/TabletMetadata.java | 48 +++
.../main/java/org/apache/accumulo/shell/Shell.java | 8 +-
.../shell/commands/ListTabletsCommand.java | 447 +
.../shell/commands/ListTabletsCommandTest.java | 193 +
.../accumulo/test/functional/TabletMetadataIT.java | 78
5 files changed, 771 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 8be6cbd..1286fce 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.core.metadata.schema;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_QUAL;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_QUAL;
@@ -29,18 +30,22 @@ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.SortedMap;
import java.util.function.Function;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
@@ -65,7 +70,12 @@ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Se
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.HostAndPort;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -75,6 +85,7 @@ import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterators;
public class TabletMetadata {
+ private static final Logger log =
LoggerFactory.getLogger(TabletMetadata.class);
private TableId tableId;
private Text prevEndRow;
@@ -435,4 +446,41 @@ public class TabletMetadata {
te.fetchedCols = EnumSet.of(ColumnType.PREV_ROW);
return te;
}
+
+ /**
+ * Get the tservers that are live from ZK. Live servers will have a valid
ZooLock. This method was
+ * pulled from org.apache.accumulo.server.master.LiveTServerSet
+ */
+ public static synchronized Set
getLiveTServers(ClientContext context) {
+final Set liveServers = new HashSet<>();
+
+final String path = context.getZooKeeperRoot() + Constants.ZTSERVERS;
+
+for (String child : context.getZooCache().getChildren(path)) {
+ checkServer(context, path, child).ifPresent(liveServers::add);
+}
+log.trace("Found {} live tservers at ZK path: {}", liveServers.size(),
path);
+
+return liveServers;
+ }
+
+ /**
+ * Check for tserver ZooLock at the ZK location. Return Optional containing
TServerInstance if a
+ * valid Zoolock exists.
+ */
+ private static Optional checkServ
[accumulo] branch 1.10 updated: Reduce log level of replication offline retries (#1843)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new 6bf3f40 Reduce log level of replication offline retries (#1843)
6bf3f40 is described below
commit 6bf3f406481a392c5bcffc15d44ccdfd52c2b577
Author: Mike Miller
AuthorDate: Wed Dec 30 07:10:41 2020 -0500
Reduce log level of replication offline retries (#1843)
* Reduce logging level to trace of retries when replication is offline
* User can easily see if repl table is online through monitor, no need
to keep spamming logs when it is offline
---
.../org/apache/accumulo/master/replication/FinishedWorkUpdater.java | 4 ++--
.../accumulo/master/replication/RemoveCompleteReplicationRecords.java | 2 +-
.../org/apache/accumulo/master/replication/ReplicationDriver.java | 2 +-
.../main/java/org/apache/accumulo/master/replication/WorkDriver.java | 2 +-
.../main/java/org/apache/accumulo/master/replication/WorkMaker.java | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/FinishedWorkUpdater.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/FinishedWorkUpdater.java
index 4495aae..01e89f2 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/replication/FinishedWorkUpdater.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/replication/FinishedWorkUpdater.java
@@ -62,10 +62,10 @@ public class FinishedWorkUpdater implements Runnable {
@Override
public void run() {
-log.debug("Looking for finished replication work");
+log.trace("Looking for finished replication work");
if (!ReplicationTable.isOnline(conn)) {
- log.debug("Replication table is not yet online, will retry");
+ log.trace("Replication table is not yet online, will retry");
return;
}
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
index 2d5f431..efa6fc4 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
@@ -74,7 +74,7 @@ public class RemoveCompleteReplicationRecords implements
Runnable {
throw new AssertionError("Inconceivable; an exception should have been"
+ " thrown, but 'bs' or 'bw' was null instead");
} catch (ReplicationTableOfflineException e) {
- log.debug("Not attempting to remove complete replication records as the"
+ log.trace("Not attempting to remove complete replication records as the"
+ " table ({}) isn't yet online", ReplicationTable.NAME);
return;
}
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
index 9ed4409..57a7adc 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
@@ -118,7 +118,7 @@ public class ReplicationDriver extends Daemon {
// Sleep for a bit
long sleepMillis =
conf.getTimeInMillis(Property.MASTER_REPLICATION_SCAN_INTERVAL);
- log.debug("Sleeping for {}ms before re-running", sleepMillis);
+ log.trace("Sleeping for {}ms before re-running", sleepMillis);
try {
Thread.sleep(sleepMillis);
} catch (InterruptedException e) {
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
index dabfc79..785addb 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
@@ -105,7 +105,7 @@ public class WorkDriver extends Daemon {
}
long sleepTime =
conf.getTimeInMillis(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP);
- log.debug("Sleeping {} ms before next work assignment", sleepTime);
+ log.trace("Sleeping {} ms before next work assignment", sleepTime);
sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
// After each loop, make sure that the WorkAssigner implementation
didn't change
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
b/server/master/src/ma
[accumulo-website] branch asf-site updated (ffca8f2 -> 18c4baf)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a change to branch asf-site
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git.
from ffca8f2 Automatic Site Publish by Buildbot
add 18c4baf Automatic Site Publish by Buildbot
No new revisions were added by this update.
Summary of changes:
output/feed.xml| 4 +-
output/release/accumulo-1.10.0/index.html | 4 +-
.../index.html | 67 --
output/release/accumulo-2.0.0-alpha-1/index.html | 4 +-
output/release/accumulo-2.0.0/index.html | 4 ++
.../{accumulo-1.3.6 => accumulo-2.0.1}/index.html | 39 -
output/release/index.html | 22 +++
output/search_data.json| 16 ++
8 files changed, 121 insertions(+), 39 deletions(-)
copy output/release/{accumulo-2.0.0-alpha-1 => accumulo-1.10.1}/index.html
(74%)
copy output/release/{accumulo-1.3.6 => accumulo-2.0.1}/index.html (84%)
[accumulo-website] branch main updated: Create drafts for 1.10.1 and 2.0.1
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 1c67ee8 Create drafts for 1.10.1 and 2.0.1
1c67ee8 is described below
commit 1c67ee89cb1cd553e012bfbddf89d01a62392254
Author: Mike Miller
AuthorDate: Wed Dec 9 10:44:05 2020 -0500
Create drafts for 1.10.1 and 2.0.1
---
_posts/release/2020-12-09-accumulo-1.10.1.md | 54
_posts/release/2020-12-09-accumulo-2.0.1.md | 27 ++
2 files changed, 81 insertions(+)
diff --git a/_posts/release/2020-12-09-accumulo-1.10.1.md
b/_posts/release/2020-12-09-accumulo-1.10.1.md
new file mode 100644
index 000..f8370bf
--- /dev/null
+++ b/_posts/release/2020-12-09-accumulo-1.10.1.md
@@ -0,0 +1,54 @@
+---
+title: Apache Accumulo 1.10.1
+sortableversion: '01.10.01'
+LTM: true
+draft: true
+---
+
+## About
+
+Apache Accumulo 1.10.1 is a bug fix release of the 1.10 LTM release line.
+
+This release contains contributions from more than X contributors from the
+Apace Accumulo community in over X commits. These release notes are highlights
of those changes. The full
+detailed changes can be seen in the git history. If anything is missing from
+this list, please [contact] us to have it included.
+
+Users of 1.10.0 or earlier are urged to upgrade to 1.10.1 as soon as it is
+available, as this is a continuation of the 1.10 LTM release line. Users are
also encouraged to
+consider migrating to a 2.x version when a suitable one becomes available.
+
+## Minimum Requirements
+
+The versions mentioned here are a guide only. It is not expected that our
+convenience binary tarball will work out-of-the-box with your particular
+environment, and some responsibility is placed on users to properly configure
+Accumulo, or even patch and rebuild it from source, for their particular
+environment.
+
+Please [contact] us or file a [bug report][issues] if you have trouble with a
+specific version or wish to seek tips. Be prepared to provide details of the
+problems you encounter, as well as perform some troubleshooting steps of your
+own, in order to get the best response.
+
+## Major Bug Fixes
+
+### TODO title bug fix
+
+## Other Miscellaneous Bug Fixes
+
+* {% ghi 1644 %} TODO Describe bugs
+
+## Useful Links
+
+* [Release VOTE email thread][vote-emails]
+* [All Changes since 1.10.0][all-changes]
+* [GitHub] - List of issues tracked on GitHub corresponding to this release
+
+[GitHub]:
https://github.com/apache/accumulo/issues?q=project%3Aapache%2Faccumulo%2F8
+[LTM]: {{ site.baseurl }}/contributor/versioning#LTM
+[all-changes]:
https://github.com/apache/accumulo/compare/rel/1.10.0...apache:rel/1.10.1
+[contact]: {{ site.baseurl }}/contact-us
+[issues]: https://github.com/apache/accumulo/issues
+[semver]: https://semver.org/spec/v2.0.0.html
+[vote-emails]:
https://lists.apache.org/thread.html/rd4731d4fd87c30958ad82a8b0be9375f2562ab0a9531ea037e646f3c%40%3Cdev.accumulo.apache.org%3E
diff --git a/_posts/release/2020-12-09-accumulo-2.0.1.md
b/_posts/release/2020-12-09-accumulo-2.0.1.md
new file mode 100644
index 000..d9852bd
--- /dev/null
+++ b/_posts/release/2020-12-09-accumulo-2.0.1.md
@@ -0,0 +1,27 @@
+---
+title: Apache Accumulo 2.0.1
+sortableversion: '02.00.01'
+draft: true
+---
+
+Apache Accumulo 2.0.1 contains bug fixes for 2.0.0. It contains X commits from
X contributors
+in the Accumulo community. The following release notes highlight some of the
+changes. If anything is missing from this list, please [contact] the developers
+to have it included.
+
+## Major Bug Fixes
+
+### TODO title bug fix
+
+## Other Miscellaneous Bug Fixes
+
+* {% ghi 1644 %} TODO Describe bugs
+
+## Upgrading
+
+View the [Upgrading Accumulo documentation][upgrade] for guidance.
+
+[contact]: /contact-us
+[semver]: https://semver.org/spec/v2.0.0.html
+[upgrade]: /docs/2.x/administration/upgrading
+[website-repo]: https://github.com/apache/accumulo-website
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit b263b863335f22b4a0f54479444b5b84ef0db9e8 Merge: 58b9de1 b8fd349 Author: Mike Miller AuthorDate: Tue Dec 8 15:38:00 2020 -0500 Merge remote-tracking branch 'upstream/1.10' into main
[accumulo] branch main updated (58b9de1 -> b263b86)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 58b9de1 Throw exceptions when permissions checks fail (#1828) add b8fd349 Improve log message in Delete CleanUp FATE. (#1829) new b263b86 Merge remote-tracking branch 'upstream/1.10' into main The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] branch 1.10 updated: Improve log message in Delete CleanUp FATE. (#1829)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new b8fd349 Improve log message in Delete CleanUp FATE. (#1829)
b8fd349 is described below
commit b8fd34907155df4d69202b95183b3934ca660d95
Author: Mike Miller
AuthorDate: Tue Dec 8 14:39:17 2020 -0500
Improve log message in Delete CleanUp FATE. (#1829)
Co-authored-by: Christopher Tubbs
---
.../src/main/java/org/apache/accumulo/master/tableOps/CleanUp.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CleanUp.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CleanUp.java
index 0adedbc..6ff79e9 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CleanUp.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CleanUp.java
@@ -102,8 +102,9 @@ class CleanUp extends MasterRepo {
if (!state.equals(TabletState.UNASSIGNED)) {
// This code will even wait on tablets that are assigned to dead
tablets servers. This is
// intentional because the master may make metadata writes for these
tablets. See #587
-log.debug("Still waiting for table to be deleted: " + tableId + "
locationState: "
-+ locationState);
+log.debug(
+"Still waiting for table({}) to be deleted; Target tablet state:
UNASSIGNED, Current tablet state: {}, locationState: {}",
+tableId, state, locationState);
done = false;
break;
}
[accumulo] branch main updated (c29c102 -> b93c678)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from c29c102 fixes #1574: remove stop / start of MAC because flushes are
waiting for completion. Reduce number of tables created so test does not time
out. (#1801)
add b93c678 Clean up some memory manager internals (#1826)
No new revisions were added by this update.
Summary of changes:
.../tabletserver/MemoryManagementActions.java | 27
.../accumulo/server/tabletserver/TabletState.java | 31 -
.../tserver/TabletServerResourceManager.java | 77 +++
.../tserver/memory}/LargestFirstMemoryManager.java | 13 +-
.../tserver/memory/TabletMemoryReport.java | 63 +
.../LargestFirstMemoryManagerTest.java | 154 -
6 files changed, 145 insertions(+), 220 deletions(-)
delete mode 100644
server/base/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManagementActions.java
delete mode 100644
server/base/src/main/java/org/apache/accumulo/server/tabletserver/TabletState.java
rename server/{base/src/main/java/org/apache/accumulo/server/tabletserver =>
tserver/src/main/java/org/apache/accumulo/tserver/memory}/LargestFirstMemoryManager.java
(96%)
create mode 100644
server/tserver/src/main/java/org/apache/accumulo/tserver/memory/TabletMemoryReport.java
rename server/tserver/src/test/java/org/apache/accumulo/tserver/{ =>
memory}/LargestFirstMemoryManagerTest.java (63%)
[accumulo] branch main updated (d33e07a -> f279ae8)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from d33e07a Fix typo in issue template (flaky, not flakey) add f279ae8 Add getTabletState to TabletMetadata (#1799) No new revisions were added by this update. Summary of changes: .../core/metadata/schema/MetadataSchema.java | 2 +- .../core/metadata/schema/TabletMetadata.java | 37 - .../core/metadata/schema/TabletsMetadata.java | 4 + .../core/metadata/schema/TabletMetadataTest.java | 89 ++ 4 files changed, 130 insertions(+), 2 deletions(-)
[accumulo] branch main updated (a7a96b8 -> a19e207)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from a7a96b8 Fix #1764 Parallelize listcompactions in shell (#1783) add a19e207 Make TabletLocationState human readable (#1796) No new revisions were added by this update. Summary of changes: .../core/metadata/TabletLocationState.java | 52 ++ 1 file changed, 24 insertions(+), 28 deletions(-)
[accumulo] branch main updated (7ee4a2d -> 5cb30ed)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 7ee4a2d Add back toString() to TServerInstance add 5cb30ed Utilize ServerContext in DeadServerList No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/server/master/state/DeadServerList.java | 5 +++-- server/manager/src/main/java/org/apache/accumulo/master/Master.java | 6 ++ .../apache/accumulo/monitor/rest/tservers/TabletServerResource.java | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-)
[accumulo] branch main updated: Add back toString() to TServerInstance
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7ee4a2d Add back toString() to TServerInstance
7ee4a2d is described below
commit 7ee4a2da58a923543495e9f10b40a5669e92f57c
Author: Mike Miller
AuthorDate: Tue Nov 17 15:53:12 2020 -0500
Add back toString() to TServerInstance
* There are places in the code that use the toString() in
TServerInstance to write to ZK, one place is WalStateManager but to be
safe need to keep the method since it is difficult to find where string
concat
is used
---
.../main/java/org/apache/accumulo/core/metadata/TServerInstance.java | 5 +
1 file changed, 5 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java
index eb7e2a3..8ba5860 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java
@@ -88,6 +88,11 @@ public class TServerInstance implements
Comparable {
return false;
}
+ @Override
+ public String toString() {
+return hostPortSession;
+ }
+
public String getHostPortSession() {
return hostPortSession;
}
[accumulo] branch main updated (7d678ab -> cccac86)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 7d678ab Update hadoop version to 3.3.0 in pom.xml (#1785) add cccac86 Refactor Tablet state classes (#1774) No new revisions were added by this update. Summary of changes: .../accumulo/core/client/ZooKeeperInstance.java| 2 +- .../accumulo/core/clientImpl/ClientContext.java| 2 +- .../core/clientImpl/RootTabletLocator.java | 2 +- .../core/clientImpl/TableOperationsImpl.java | 2 +- .../apache/accumulo/core/logging/TabletLogger.java | 12 +-- .../accumulo/core/metadata}/SuspendingTServer.java | 6 +- .../accumulo/core/metadata}/TServerInstance.java | 67 ++-- .../core/metadata}/TabletLocationState.java| 4 +- .../accumulo/core/metadata}/TabletState.java | 2 +- .../accumulo/core/metadata/schema/Ample.java | 18 + .../core/metadata/schema/TabletMetadata.java | 30 +--- .../org/apache/accumulo/core/summary/Gatherer.java | 4 +- .../accumulo/server/log/WalStateManager.java | 2 +- .../accumulo/server/master/LiveTServerSet.java | 4 +- .../master/balancer/ChaoticLoadBalancer.java | 2 +- .../master/balancer/DefaultLoadBalancer.java | 6 +- .../server/master/balancer/GroupBalancer.java | 88 ++ .../balancer/HostRegexTableLoadBalancer.java | 8 +- .../server/master/balancer/TableLoadBalancer.java | 2 +- .../server/master/balancer/TabletBalancer.java | 4 +- .../accumulo/server/master/state/Assignment.java | 1 + .../accumulo/server/master/state/CurrentState.java | 1 + .../master/state/LoggingTabletStateStore.java | 4 +- .../server/master/state/MetaDataStateStore.java| 2 + .../server/master/state/MetaDataTableScanner.java | 5 +- .../server/master/state/RootTabletStateStore.java | 1 + .../server/master/state/TabletMigration.java | 1 + .../master/state/TabletStateChangeIterator.java| 6 +- .../server/master/state/TabletStateStore.java | 2 + .../server/master/state/ZooTabletStateStore.java | 13 ++-- .../server/metadata/TabletMutatorBase.java | 13 ++-- .../accumulo/server/util/FindOfflineTablets.java | 6 +- .../accumulo/server/util/MasterMetadataUtil.java | 2 +- .../accumulo/server/master/LiveTServerSetTest.java | 2 +- .../BaseHostRegexTableLoadBalancerTest.java| 4 +- .../master/balancer/ChaoticLoadBalancerTest.java | 2 +- .../master/balancer/DefaultLoadBalancerTest.java | 2 +- .../server/master/balancer/GroupBalancerTest.java | 13 ++-- ...tRegexTableLoadBalancerReconfigurationTest.java | 14 ++-- .../balancer/HostRegexTableLoadBalancerTest.java | 35 + .../master/balancer/TableLoadBalancerTest.java | 7 +- .../master/state/RootTabletStateStoreTest.java | 4 +- .../master/state/TabletLocationStateTest.java | 3 + .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 6 +- .../gc/GarbageCollectWriteAheadLogsTest.java | 4 +- .../java/org/apache/accumulo/master/Master.java| 18 ++--- .../master/MasterClientServiceHandler.java | 8 +- .../apache/accumulo/master/TabletGroupWatcher.java | 10 +-- .../replication/MasterReplicationCoordinator.java | 9 +-- .../apache/accumulo/master/state/MergeStats.java | 6 +- .../apache/accumulo/master/state/TableCounts.java | 2 +- .../apache/accumulo/master/state/TableStats.java | 2 +- .../master/tableOps/bulkVer1/CopyFailed.java | 2 +- .../master/tableOps/bulkVer1/LoadFiles.java| 6 +- .../master/tableOps/compact/CompactionDriver.java | 4 +- .../accumulo/master/tableOps/delete/CleanUp.java | 4 +- .../master/tserverOps/ShutdownTServer.java | 18 +++-- .../accumulo/master/upgrade/Upgrader9to10.java | 2 +- .../MasterReplicationCoordinatorTest.java | 2 +- .../master/tableOps/ShutdownTServerTest.java | 16 ++-- .../monitor/rest/tables/TablesResource.java| 4 +- .../org/apache/accumulo/tserver/TabletServer.java | 5 +- .../accumulo/tserver/UnloadTabletHandler.java | 6 +- .../accumulo/tserver/tablet/DatafileManager.java | 2 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +- .../apache/accumulo/tserver/tablet/TabletData.java | 4 +- .../accumulo/tserver/CheckTabletMetadataTest.java | 2 +- .../test/MasterRepairsDualAssignmentIT.java| 4 +- .../java/org/apache/accumulo/test/UnusedWALIT.java | 2 +- .../test/functional/MasterAssignmentIT.java| 2 +- .../test/functional/RegexGroupBalanceIT.java | 6 +- .../accumulo/test/functional/SplitRecoveryIT.java | 2 +- .../functional/TabletStateChangeIteratorIT.java| 2 +- .../apache/accumulo/test/master/MergeStateIT.java | 4 +- .../accumulo/test/master/SuspendedTabletsIT.java | 8 +- .../accumulo/test/performance/NullTserver.java | 4
[accumulo] branch main updated (fe4d4d1 -> 1861473)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from fe4d4d1 Improve tool tip for hold time in Monitor (#1772) add 1861473 Make cluster script start monitor last. Fixes #1777 (#1778) No new revisions were added by this update. Summary of changes: assemble/bin/accumulo-cluster | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[accumulo] branch main updated: Improve tool tip for hold time in Monitor (#1772)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new fe4d4d1 Improve tool tip for hold time in Monitor (#1772) fe4d4d1 is described below commit fe4d4d1f8d9a7c62a5372b6adacfa6d9f55a9900 Author: Mike Miller AuthorDate: Thu Nov 12 11:02:16 2020 -0500 Improve tool tip for hold time in Monitor (#1772) * Also make the description for Ingest the same on all pages --- .../main/resources/org/apache/accumulo/monitor/templates/table.ftl | 2 +- .../main/resources/org/apache/accumulo/monitor/templates/tables.ftl | 2 +- .../resources/org/apache/accumulo/monitor/templates/tservers.ftl| 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/table.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/table.ftl index 5506a1a..891589d 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/table.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/table.ftl @@ -44,7 +44,7 @@ Entries Ingest Query -HoldTime +HoldTime RunningScans MinorCompactions MajorCompactions diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tables.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tables.ftl index 38212f2..d4b232b 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tables.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tables.ftl @@ -101,7 +101,7 @@ Ingest Read Returned - HoldTime + HoldTime Scans MinC MajC diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tservers.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tservers.ftl index 2066f24..291595a 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tservers.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/tservers.ftl @@ -57,15 +57,15 @@ LastContact ResponseTime Entries -Ingest +Ingest Query -HoldTime +HoldTime RunningScans MinorCompactions MajorCompactions Index CacheHit Rate Data CacheHit Rate -OSLoad +OSLoad
[accumulo] branch main updated: Minor cleanup in BulkImport (#1769)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 46ee51d Minor cleanup in BulkImport (#1769)
46ee51d is described below
commit 46ee51d0c6afe9910873f55653017b91c45a345a
Author: Mike Miller
AuthorDate: Wed Nov 4 16:35:25 2020 -0500
Minor cleanup in BulkImport (#1769)
* Remove exceptions from BulkImport and ConcurrentKeyExtentCache
internal classes that were declared but never thrown
* Remove other unused variables
* Shorten a few lines that use lambdas
---
.../accumulo/core/clientImpl/bulk/BulkImport.java | 35 ++
.../clientImpl/bulk/ConcurrentKeyExtentCache.java | 9 ++
.../bulk/ConcurrentKeyExtentCacheTest.java | 13 ++--
3 files changed, 14 insertions(+), 43 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
index 1a60318..160db69 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
@@ -22,6 +22,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.stream.Collectors.groupingBy;
+import static
org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.pathToCacheId;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -75,7 +76,6 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.fate.util.Retry;
@@ -302,35 +302,25 @@ public class BulkImport implements
ImportDestinationArguments, ImportMappingOpti
}
public interface KeyExtentCache {
-KeyExtent lookup(Text row)
-throws AccumuloException, AccumuloSecurityException,
TableNotFoundException;
+KeyExtent lookup(Text row);
}
public static List findOverlappingTablets(KeyExtentCache
extentCache,
- FileSKVIterator reader)
- throws IOException, AccumuloException, AccumuloSecurityException,
TableNotFoundException {
-
-Text startRow = null;
-Text endRow = null;
+ FileSKVIterator reader) throws IOException {
List result = new ArrayList<>();
Collection columnFamilies = Collections.emptyList();
-Text row = startRow;
-if (row == null)
- row = new Text();
+Text row = new Text();
while (true) {
- // log.debug(filename + " Seeking to row " + row);
reader.seek(new Range(row, null), columnFamilies, false);
if (!reader.hasTop()) {
-// log.debug(filename + " not found");
break;
}
row = reader.getTopKey().getRow();
KeyExtent extent = extentCache.lookup(row);
- // log.debug(filename + " found row " + row + " at location " +
tabletLocation);
result.add(extent);
row = extent.endRow();
- if (row != null && (endRow == null || row.compareTo(endRow) < 0)) {
+ if (row != null) {
row = nextRow(row);
} else
break;
@@ -347,8 +337,7 @@ public class BulkImport implements
ImportDestinationArguments, ImportMappingOpti
public static List findOverlappingTablets(ClientContext context,
KeyExtentCache extentCache, Path file, FileSystem fs, Cache
fileLenCache,
- CryptoService cs)
- throws IOException, AccumuloException, AccumuloSecurityException,
TableNotFoundException {
+ CryptoService cs) throws IOException {
try (FileSKVIterator reader =
FileOperations.getInstance().newReaderBuilder()
.forFile(file.toString(), fs, fs.getConf(), cs)
.withTableConfiguration(context.getConfiguration()).withFileLenCache(fileLenCache)
@@ -361,7 +350,6 @@ public class BulkImport implements
ImportDestinationArguments, ImportMappingOpti
HashMap fileLens = new HashMap<>();
for (FileStatus status : statuses) {
fileLens.put(status.getPath().getName(), status.getLen());
- status.getLen();
}
return fileLens;
@@ -371,9 +359,7 @@ public class BulkImport implements
ImportDestinationArguments, ImportMappingOpti
Map fileLens = getFileLenMap(statuses);
Map absFileLens = new HashMap<>();
-fileLens.forEach((k, v) -> {
- absFileLens.put(CachableBlockFile.pathToCacheId(new Path(dir, k)), v);
-});
+
[accumulo] branch main updated: Modify IteratorEnvIT to expose bug
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 02f778a Modify IteratorEnvIT to expose bug
02f778a is described below
commit 02f778a87bc74659dac45b9caf278bd7267e0dad
Author: Mike Miller
AuthorDate: Tue Nov 3 13:57:46 2020 -0500
Modify IteratorEnvIT to expose bug
* IteratorEnvIT had flawed logic that was masking a bug in the
implementation of IteratorEnvironment
* IteratorEnvironment.getPluginEnv() is not impl and currently throws an
UnsupportedOperationException
---
test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
b/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
index f64afe3..6965b69 100644
--- a/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/IteratorEnvIT.java
@@ -146,10 +146,14 @@ public class IteratorEnvIT extends AccumuloClusterHarness
{
private static void testEnv(IteratorScope scope, Map opts,
IteratorEnvironment env) {
TableId expectedTableId = TableId.of(opts.get("expected.table.id"));
-if
(!"value1".equals(env.getConfig().get("table.custom.iterator.env.test")) &&
!"value1".equals(
+if
(!"value1".equals(env.getConfig().get("table.custom.iterator.env.test")))
+ throw new RuntimeException("Test failed - Expected table property not
found.");
+if (!"value1".equals(
env.getServiceEnv().getConfiguration(env.getTableId()).getTableCustom("iterator.env.test")))
throw new RuntimeException("Test failed - Expected table property not
found.");
-if
(!"value1".equals(env.getConfig().get("table.custom.iterator.env.test")) &&
!"value1".equals(
+if
(!"value1".equals(env.getConfig().get("table.custom.iterator.env.test")))
+ throw new RuntimeException("Test failed - Expected table property not
found.");
+if (!"value1".equals(
env.getPluginEnv().getConfiguration(env.getTableId()).getTableCustom("iterator.env.test")))
throw new RuntimeException("Test failed - Expected table property not
found.");
if (!scope.equals(env.getIteratorScope()))
[accumulo] branch main updated: Refactor TabletGroupWatcher (#1761)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3f47da1 Refactor TabletGroupWatcher (#1761)
3f47da1 is described below
commit 3f47da1777f5d059d42d7706574ee3fd9882006b
Author: Mike Miller
AuthorDate: Mon Nov 2 10:14:14 2020 -0500
Refactor TabletGroupWatcher (#1761)
* Create TabletLists to hold the many different data structures being
tracked in the run method of TabletGroupWatcher
* Create methods for some of the logic in the switch case
* Pass TabletLists to the flush method and break flush into methods
* Rename TabletLocationState.getServer() to getLocation() as location is
a more meaningful name instead of server
* Move markDeadServerLogsAsClosed from Master and make private
---
.../server/master/state/TabletLocationState.java | 2 +-
.../master/state/TabletLocationStateTest.java | 8 +-
.../java/org/apache/accumulo/master/Master.java| 13 -
.../apache/accumulo/master/TabletGroupWatcher.java | 279 -
4 files changed, 171 insertions(+), 131 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
index c3a96aa..c2111ef 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
@@ -87,7 +87,7 @@ public class TabletLocationState {
return extent + "@(" + future + "," + current + "," + last + ")" +
(chopped ? " chopped" : "");
}
- public TServerInstance getServer() {
+ public TServerInstance getLocation() {
TServerInstance result = null;
if (current != null) {
result = current;
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/master/state/TabletLocationStateTest.java
b/server/base/src/test/java/org/apache/accumulo/server/master/state/TabletLocationStateTest.java
index 9b9d148..b31d138 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/master/state/TabletLocationStateTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/master/state/TabletLocationStateTest.java
@@ -104,25 +104,25 @@ public class TabletLocationStateTest {
@Test
public void testGetServer_Current() throws Exception {
tls = new TabletLocationState(keyExtent, null, current, last, null,
walogs, true);
-assertSame(current, tls.getServer());
+assertSame(current, tls.getLocation());
}
@Test
public void testGetServer_Future() throws Exception {
tls = new TabletLocationState(keyExtent, future, null, last, null, walogs,
true);
-assertSame(future, tls.getServer());
+assertSame(future, tls.getLocation());
}
@Test
public void testGetServer_Last() throws Exception {
tls = new TabletLocationState(keyExtent, null, null, last, null, walogs,
true);
-assertSame(last, tls.getServer());
+assertSame(last, tls.getLocation());
}
@Test
public void testGetServer_None() throws Exception {
tls = new TabletLocationState(keyExtent, null, null, null, null, walogs,
true);
-assertNull(tls.getServer());
+assertNull(tls.getLocation());
}
@Test
diff --git
a/server/manager/src/main/java/org/apache/accumulo/master/Master.java
b/server/manager/src/main/java/org/apache/accumulo/master/Master.java
index bacc583..45449f7 100644
--- a/server/manager/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/manager/src/main/java/org/apache/accumulo/master/Master.java
@@ -100,8 +100,6 @@ import org.apache.accumulo.server.HighlyAvailableService;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.ServerOpts;
import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.log.WalStateManager;
-import org.apache.accumulo.server.log.WalStateManager.WalMarkerException;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
@@ -135,7 +133,6 @@ import org.apache.accumulo.server.util.TableInfoUtil;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.accumulo.start.classloader.vfs.ContextManager;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.thrift.TException;
@@ -1675,16 +1672,6 @@ public class Master extends AbstractServer
}
}
- public void markDe
[accumulo] branch main updated: Create Ample interface method for createDeleteMutation (#1757)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d820360 Create Ample interface method for createDeleteMutation (#1757)
d820360 is described below
commit d820360b975de759b1575d68456e69c828225a85
Author: Mike Miller
AuthorDate: Fri Oct 30 12:26:48 2020 -0400
Create Ample interface method for createDeleteMutation (#1757)
* Drop static method in favor of method on the Ample interface for
creating delete mutations
* Change GarbageCollectorIT to use Ample for one of the tests
* Reuse Ample object in loops to prevent extra object creation
* Further improvements could be made to not have the Mutation type in Ample
---
.../org/apache/accumulo/core/metadata/TabletFileUtil.java | 3 ++-
.../org/apache/accumulo/core/metadata/schema/Ample.java | 15 +++
.../apache/accumulo/server/metadata/ServerAmpleImpl.java | 11 ++-
.../apache/accumulo/server/util/MetadataTableUtil.java| 14 --
.../org/apache/accumulo/master/TabletGroupWatcher.java| 13 -
.../org/apache/accumulo/master/upgrade/Upgrader9to10.java | 4 ++--
.../accumulo/test/functional/GarbageCollectorIT.java | 6 +++---
7 files changed, 44 insertions(+), 22 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
index 372c017..ff92ae3 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
@@ -28,7 +28,8 @@ public class TabletFileUtil {
/**
* Validate if string is a valid path. Return normalized string or throw
exception if not valid.
* This was added to facilitate more use of TabletFile over String but this
puts the validation in
- * one location in the case where TabletFile can't be used.
+ * one location in the case where TabletFile can't be used. The Garbage
Collector is optimized to
+ * store a directory for Tablet File so a String is used.
*/
public static String validate(String path) {
Path p = new Path(path);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 2a33e17..4703b32 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.metadata.schema;
import java.util.Collection;
import java.util.Iterator;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
@@ -152,6 +153,20 @@ public interface Ample {
}
/**
+ * Return an encoded delete marker Mutation to delete the specified
TabletFile path. A String is
+ * used for the parameter because the Garbage Collector is optimized to
store a directory for
+ * Tablet File. Otherwise a {@link TabletFile} object could be used. The
tabletFilePathToRemove is
+ * validated and normalized before creating the mutation.
+ *
+ * @param tabletFilePathToRemove
+ * String full path of the TabletFile
+ * @return Mutation with encoded delete marker
+ */
+ default Mutation createDeleteMutation(String tabletFilePathToRemove) {
+throw new UnsupportedOperationException();
+ }
+
+ /**
* This interface allows efficiently updating multiple tablets. Unless close
is called, changes
* may not be persisted.
*/
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index c42715e..b1d89dd 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -195,18 +195,19 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
}
}
- public static Mutation createDeleteMutation(String pathToRemove) {
-String path = TabletFileUtil.validate(pathToRemove);
+ @Override
+ public Mutation createDeleteMutation(String tabletFilePathToRemove) {
+String path = TabletFileUtil.validate(tabletFilePathToRemove);
return createDelMutation(path);
}
- public static Mutation createDeleteMutation(StoredTabletFile pathToRemove) {
+ public Mutation createDeleteMutation(StoredTabletFile pathToRemove) {
return createDelMutation(pathToRemove.getMetaUpdateDelete());
}
- private static Mutation createDelMutation(String path) {
+ private Mutation
[accumulo] branch 1.10 updated (51dde75 -> 252897d)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch 1.10 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 51dde75 Configure surefire/failsafe forkCount separately (#1756) add 252897d Back port #1754 to 1.10 No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[accumulo] branch 1.10 updated (51dde75 -> 252897d)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch 1.10 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 51dde75 Configure surefire/failsafe forkCount separately (#1756) add 252897d Back port #1754 to 1.10 No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
