[accumulo] branch main updated: Fixes for new Admin fate command (#2987)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a57cec3dc8 Fixes for new Admin fate command (#2987)
a57cec3dc8 is described below
commit a57cec3dc800ff0c9e20ba27dc66743f5969872a
Author: Mike Miller
AuthorDate: Fri Sep 30 19:16:17 2022 +
Fixes for new Admin fate command (#2987)
* A few fixes for new Admin fate command
* Make Admin fate conform to Git command usage guidelines
* Closes #2974
---
.../org/apache/accumulo/server/util/Admin.java | 35 ++
1 file changed, 16 insertions(+), 19 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index e6df9ff0af..2f5010846e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -68,6 +68,7 @@ import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.tables.TableMap;
import org.apache.accumulo.fate.AdminUtil;
+import org.apache.accumulo.fate.FateTxId;
import org.apache.accumulo.fate.ReadOnlyTStore;
import org.apache.accumulo.fate.ZooStore;
import org.apache.accumulo.fate.zookeeper.ServiceLock;
@@ -228,19 +229,19 @@ public class Admin implements KeywordExecutable {
List txList = new ArrayList<>();
@Parameter(names = {"-c", "--cancel"},
-description = "[ ...] Cancel new or submitted FaTE
transactions")
+description = "... Cancel new or submitted FaTE transactions")
boolean cancel;
@Parameter(names = {"-f", "--fail"},
-description = "[ ...] Transition FaTE transaction status
to FAILED_IN_PROGRESS (requires Manager to be down)")
+description = "... Transition FaTE transaction status to
FAILED_IN_PROGRESS (requires Manager to be down)")
boolean fail;
@Parameter(names = {"-d", "--delete"},
-description = "[ ...] Delete locks associated with
transactions (Requires Manager to be down)")
+description = "... Delete locks associated with transactions
(Requires Manager to be down)")
boolean delete;
-@Parameter(names = {"-p", "--print", "-l", "--list"},
-description = "[txId ...] Print information about FaTE
transactions. Print only the 'txId's specified or print all transactions if
empty. Use -s to only print certain states.")
+@Parameter(names = {"-p", "--print", "-print", "-l", "--list", "-list"},
+description = "[...] Print information about FaTE transactions.
Print only the 'txId's specified or print all transactions if empty. Use -s to
only print certain states.")
boolean print;
@Parameter(names = "--summary", description = "Print a summary of all FaTE
transactions")
@@ -250,7 +251,7 @@ public class Admin implements KeywordExecutable {
boolean printJson;
@Parameter(names = {"-s", "--state"},
-description = "[ -s ...] Print transactions in the
state(s) {NEW, IN_PROGRESS, FAILED_IN_PROGRESS, FAILED, SUCCESSFUL}")
+description = "... Print transactions in the state(s) {NEW,
IN_PROGRESS, FAILED_IN_PROGRESS, FAILED, SUCCESSFUL}")
List states = new ArrayList<>();
}
@@ -282,9 +283,6 @@ public class Admin implements KeywordExecutable {
JCommander cl = new JCommander(opts);
cl.setProgramName("accumulo admin");
-FateOpsCommand fateOpsCommand = new FateOpsCommand();
-cl.addCommand("fate", fateOpsCommand);
-
ChangeSecretCommand changeSecretCommand = new ChangeSecretCommand();
cl.addCommand("changeSecret", changeSecretCommand);
@@ -297,6 +295,9 @@ public class Admin implements KeywordExecutable {
DumpConfigCommand dumpConfigCommand = new DumpConfigCommand();
cl.addCommand("dumpConfig", dumpConfigCommand);
+FateOpsCommand fateOpsCommand = new FateOpsCommand();
+cl.addCommand("fate", fateOpsCommand);
+
ListInstancesCommand listInstancesOpts = new ListInstancesCommand();
cl.addCommand("listInstances", listInstancesOpts);
@@ -797,13 +798,8 @@ public class Admin implements KeywordExecutable {
if (fateOpsCommand.print) {
final Set sortedTxs = new TreeSet<>();
fateOpsCommand.txList.forEach(s ->
sortedTxs.add(parseTidFromUserInput(s)));
- if (!fateOpsCommand.txList.isEmpty()) {
-EnumSet status
[accumulo] branch main updated (bbdfa49291 -> ff58343c21)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from bbdfa49291 Removed API method from InstanceOperations that exposed Thrift object (#2985) add ff58343c21 Drop ReadOnlyStore (#2976) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/fate/ReadOnlyStore.java| 124 - .../apache/accumulo/fate/ReadOnlyStoreTest.java| 76 - .../org/apache/accumulo/server/util/Admin.java | 12 +- .../manager/upgrade/UpgradeCoordinator.java| 6 +- 4 files changed, 6 insertions(+), 212 deletions(-) delete mode 100644 core/src/main/java/org/apache/accumulo/fate/ReadOnlyStore.java delete mode 100644 core/src/test/java/org/apache/accumulo/fate/ReadOnlyStoreTest.java
[accumulo] branch main updated: Partial revert changes from #1519 (#2984)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1a42c54cff Partial revert changes from #1519 (#2984)
1a42c54cff is described below
commit 1a42c54cff170037b244969ff2a40797ea50ce1e
Author: Mike Miller
AuthorDate: Thu Sep 29 16:46:50 2022 +
Partial revert changes from #1519 (#2984)
* Revert some changes to FileUtil that used TabletFile. If a tablet is
splitting and exceeds the tserver.tablet.split.midpoint.files.max then
we will reduce the number of index files using temporary files. These
temporary files don't conform to standard TabletFile directories so just
use strings in this special case.
* Closes #2977
---
.../org/apache/accumulo/server/util/FileUtil.java | 58 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 5 +-
2 files changed, 37 insertions(+), 26 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 3e6cebb523..a945655d90 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
@@ -108,11 +109,11 @@ public class FileUtil {
return result;
}
- public static Collection reduceFiles(ServerContext context,
- TableConfiguration tableConf, Text prevEndRow, Text endRow,
Collection mapFiles,
- int maxFiles, Path tmpDir, int pass) throws IOException {
+ public static Collection reduceFiles(ServerContext context,
TableConfiguration tableConf,
+ Text prevEndRow, Text endRow, Collection mapFiles, int maxFiles,
Path tmpDir,
+ int pass) throws IOException {
-ArrayList paths = new ArrayList<>(mapFiles);
+ArrayList paths = new ArrayList<>(mapFiles);
if (paths.size() <= maxFiles)
return paths;
@@ -121,33 +122,33 @@ public class FileUtil {
int start = 0;
-ArrayList outFiles = new ArrayList<>();
+ArrayList outFiles = new ArrayList<>();
int count = 0;
while (start < paths.size()) {
int end = Math.min(maxFiles + start, paths.size());
- List inFiles = paths.subList(start, end);
+ List inFiles = paths.subList(start, end);
start = end;
- TabletFile newMapFile =
- new TabletFile(new Path(String.format("%s/%04d.%s", newDir, count++,
RFile.EXTENSION)));
+ // temporary tablet file does not conform to typical path verified in
TabletFile
+ String newMapFile = String.format("%s/%04d.%s", newDir, count++,
RFile.EXTENSION);
outFiles.add(newMapFile);
- FileSystem ns =
context.getVolumeManager().getFileSystemByPath(newMapFile.getPath());
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(new
Path(newMapFile));
FileSKVWriter writer = new RFileOperations().newWriterBuilder()
- .forFile(newMapFile.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(newMapFile, ns, ns.getConf(), tableConf.getCryptoService())
.withTableConfiguration(tableConf).build();
writer.startDefaultLocalityGroup();
List> iters = new
ArrayList<>(inFiles.size());
FileSKVIterator reader = null;
try {
-for (TabletFile file : inFiles) {
- ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
+for (String file : inFiles) {
+ ns = context.getVolumeManager().getFileSystemByPath(new Path(file));
reader = FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(file, ns, ns.getConf(), tableConf.getCryptoService())
.withTableConfiguration(tableConf).build();
iters.add(reader);
}
@@ -199,8 +200,8 @@ public class FileUtil {
}
public static double estimatePercentageLTE(ServerContext context,
TableConfiguration tableConf,
- String tabletDir, Text prevEndRow, Text endRow, Collection
mapFiles,
- Text splitRow) throws IOException {
+ String tabletDir, Text prevEndRow, Text endRow, Collection
mapFiles, Text splitRow)
+ throws IOException {
Path tmpDir = null;
@@ -277,9 +278,9 @@ public class FileUtil {
*/
public static SortedMap findMidPoint(ServerContext context,
TableConfiguration tableConf, String tabletDirectory, Text prevEndRow,
Text endRow,
- Coll
[accumulo] branch main updated: Reduce compaction error to debug (#2981)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9fc349603e Reduce compaction error to debug (#2981)
9fc349603e is described below
commit 9fc349603e3d5f5c52fcea632f0bd80e022e5ff6
Author: Mike Miller
AuthorDate: Thu Sep 29 13:19:10 2022 +
Reduce compaction error to debug (#2981)
* Closes #2978
---
.../java/org/apache/accumulo/tserver/tablet/CompactableImpl.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index e7ec5856ab..e4660ea964 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -439,8 +439,11 @@ public class CompactableImpl implements Compactable {
Set candidates = new
HashSet<>(selectedFiles);
candidates.removeAll(allCompactingFiles);
candidates = Collections.unmodifiableSet(candidates);
-Preconditions.checkState(currFiles.containsAll(candidates),
-"selected files not in all files %s %s", candidates,
currFiles);
+// verify that candidates are still around and fail quietly if
not
+if (!currFiles.containsAll(candidates)) {
+ log.debug("Selected files not in all files {} {}",
candidates, currFiles);
+ return Set.of();
+}
return candidates;
} else {
return Set.of();
[accumulo] branch main updated: Fix NPE in suspend logging (#2970)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1a3c7c1fc7 Fix NPE in suspend logging (#2970)
1a3c7c1fc7 is described below
commit 1a3c7c1fc75dcf665946d2d3dcbb36c52528f0f2
Author: Mike Miller
AuthorDate: Wed Sep 28 13:09:49 2022 +
Fix NPE in suspend logging (#2970)
---
.../accumulo/server/manager/state/LoggingTabletStateStore.java | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
index 9b77eca49c..4a9fdb7845 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
+import org.apache.accumulo.core.util.HostAndPort;
import org.apache.hadoop.fs.Path;
/**
@@ -85,8 +86,13 @@ class LoggingTabletStateStore implements TabletStateStore {
logsForDeadServers = Map.of();
for (TabletLocationState tls : tablets) {
- TabletLogger.suspended(tls.extent, tls.current.getHostAndPort(),
suspensionTimestamp,
- TimeUnit.MILLISECONDS, logsForDeadServers.size());
+ var location = tls.getLocation();
+ HostAndPort server = null;
+ if (location != null) {
+server = location.getHostAndPort();
+ }
+ TabletLogger.suspended(tls.extent, server, suspensionTimestamp,
TimeUnit.MILLISECONDS,
+ logsForDeadServers.size());
}
}
[accumulo-testing] branch 2.0 updated: Fix Bulk Rwalk (#230)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/2.0 by this push:
new 43796e6 Fix Bulk Rwalk (#230)
43796e6 is described below
commit 43796e665b2b45b7f1d5938cd41b221369d1edf2
Author: Mike Miller
AuthorDate: Mon Sep 26 15:55:40 2022 +
Fix Bulk Rwalk (#230)
---
.../java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
index 55d81c1..5d6398e 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
@@ -94,7 +94,7 @@ public class BulkPlusOne extends BulkImportTest {
writer.close();
}
env.getAccumuloClient().tableOperations().importDirectory(dir.toString())
-.to(Setup.getTableName()).tableTime(true);
+.to(Setup.getTableName()).tableTime(true).load();
fs.delete(dir, true);
log.debug("Finished bulk import, start rows " + printRows + " last row "
+ String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier);
[accumulo] branch main updated: Revert "Modify FateMetrics to enforce ReadOnlyStore required for ZooKeeper (#2810)" (#2971)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5d37b28e08 Revert "Modify FateMetrics to enforce ReadOnlyStore
required for ZooKeeper (#2810)" (#2971)
5d37b28e08 is described below
commit 5d37b28e08447fb6aea3d42b962483bd86a3f5bc
Author: Mike Miller
AuthorDate: Wed Sep 28 10:54:58 2022 +
Revert "Modify FateMetrics to enforce ReadOnlyStore required for ZooKeeper
(#2810)" (#2971)
This reverts commit 76092fc58eab7a3d4cc1fb87c376dc25b6f87537.
---
.../org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 11 +--
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
index c34e0e5f64..99398d5a03 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
@@ -28,7 +28,6 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
-import org.apache.accumulo.fate.ReadOnlyStore;
import org.apache.accumulo.fate.ReadOnlyTStore;
import org.apache.accumulo.fate.ZooStore;
import org.apache.accumulo.server.ServerContext;
@@ -50,7 +49,7 @@ public class FateMetrics implements MetricsProducer {
private static final String OP_TYPE_TAG = "op.type";
private final ServerContext context;
- private final ReadOnlyStore zooStore;
+ private final ReadOnlyTStore zooStore;
private final String fateRootPath;
private final long refreshDelay;
@@ -72,8 +71,7 @@ public class FateMetrics implements MetricsProducer {
this.refreshDelay = Math.max(DEFAULT_MIN_REFRESH_DELAY,
minimumRefreshDelay);
try {
- this.zooStore =
- new ReadOnlyStore<>(new ZooStore<>(fateRootPath,
context.getZooReaderWriter()));
+ this.zooStore = new ZooStore<>(fateRootPath,
context.getZooReaderWriter());
} catch (KeeperException ex) {
throw new IllegalStateException(
"FATE Metrics - Failed to create zoo store - metrics unavailable",
ex);
@@ -122,8 +120,9 @@ public class FateMetrics implements MetricsProducer {
}
}
-metricValues.getOpTypeCounters().forEach((name, count) -> Metrics
-.gauge(METRICS_FATE_TYPE_IN_PROGRESS, Tags.of(OP_TYPE_TAG, name),
count));
+metricValues.getOpTypeCounters().forEach((name, count) -> {
+ Metrics.gauge(METRICS_FATE_TYPE_IN_PROGRESS, Tags.of(OP_TYPE_TAG, name),
count);
+});
}
@Override
[accumulo] branch main updated: Add missing chopped column to Ample (#2969)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 941c596866 Add missing chopped column to Ample (#2969)
941c596866 is described below
commit 941c596866e4a438a2c9525f4051900796f81fe2
Author: Mike Miller
AuthorDate: Tue Sep 27 18:48:31 2022 +
Add missing chopped column to Ample (#2969)
* Fixes #2961
---
.../apache/accumulo/core/metadata/schema/MetadataSchema.java | 5 +++--
.../apache/accumulo/core/metadata/schema/TabletMetadata.java | 11 +++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 5f058bc6c7..eb39ac71d8 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -326,8 +326,9 @@ public class MetadataSchema {
* data for the current tablet, so that they are safe to merge
*/
public static class ChoppedColumnFamily {
- public static final Text NAME = new Text("chopped");
- public static final ColumnFQ CHOPPED_COLUMN = new ColumnFQ(NAME, new
Text("chopped"));
+ public static final String STR_NAME = "chopped";
+ public static final Text NAME = new Text(STR_NAME);
+ public static final ColumnFQ CHOPPED_COLUMN = new ColumnFQ(NAME, new
Text(STR_NAME));
}
public static class ExternalCompactionColumnFamily {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 5305858d5d..89d7722958 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -54,6 +54,7 @@ import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.TabletLocationState;
import org.apache.accumulo.core.metadata.TabletState;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
@@ -106,6 +107,7 @@ public class TabletMetadata {
private OptionalLong compact = OptionalLong.empty();
private Double splitRatio = null;
private Map extCompactions;
+ private boolean chopped = false;
public enum LocationType {
CURRENT, FUTURE, LAST
@@ -127,6 +129,7 @@ public class TabletMetadata {
COMPACT_ID,
SPLIT_RATIO,
SUSPEND,
+CHOPPED,
ECOMP
}
@@ -265,6 +268,11 @@ public class TabletMetadata {
return splitRatio;
}
+ public boolean hasChopped() {
+ensureFetched(ColumnType.CHOPPED);
+return chopped;
+ }
+
public SortedMap getKeyValues() {
Preconditions.checkState(keyValues != null, "Requested key values when it
was not saved");
return keyValues;
@@ -399,6 +407,9 @@ public class TabletMetadata {
extCompBuilder.put(ExternalCompactionId.of(qual),
ExternalCompactionMetadata.fromJson(val));
break;
+case ChoppedColumnFamily.STR_NAME:
+ te.chopped = true;
+ break;
default:
throw new IllegalStateException("Unexpected family " + fam);
}
[accumulo-testing] branch main updated: Fix Bulk Rwalk (#230)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new cfefe01 Fix Bulk Rwalk (#230)
cfefe01 is described below
commit cfefe01f6a2264efa2add9361745408ce2eafedb
Author: Mike Miller
AuthorDate: Mon Sep 26 15:55:40 2022 +
Fix Bulk Rwalk (#230)
---
.../java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
index 872d58c..e13e259 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
@@ -91,7 +91,7 @@ public class BulkPlusOne extends BulkImportTest {
}
}
env.getAccumuloClient().tableOperations().importDirectory(dir.toString())
-.to(Setup.getTableName()).tableTime(true);
+.to(Setup.getTableName()).tableTime(true).load();
fs.delete(dir, true);
log.debug("Finished bulk import, start rows " + printRows + " last row "
+ String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier);
[accumulo] branch main updated: Fix crypto in Upgrader9to10 (#2959)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 31046704d3 Fix crypto in Upgrader9to10 (#2959)
31046704d3 is described below
commit 31046704d39b1cad3f682505bed09da000a2906d
Author: Mike Miller
AuthorDate: Mon Sep 26 13:36:15 2022 +
Fix crypto in Upgrader9to10 (#2959)
* Closes #2960
---
.../main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
index 96315e08ba..fc99d90037 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
@@ -69,6 +69,7 @@ import
org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
+import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
@@ -430,7 +431,7 @@ public class Upgrader9to10 implements Upgrader {
var tableConf = context.getTableConfiguration(RootTable.ID);
long maxTime = -1;
try (FileSKVIterator reader =
FileOperations.getInstance().newReaderBuilder()
-.forFile(path.toString(), ns, ns.getConf(),
tableConf.getCryptoService())
+.forFile(path.toString(), ns, ns.getConf(),
NoCryptoServiceFactory.NONE)
.withTableConfiguration(tableConf).seekToBeginning().build()) {
while (reader.hasTop()) {
maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
[accumulo] branch main updated: Drop FateAdmin classes (#2947)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new acd457ace6 Drop FateAdmin classes (#2947)
acd457ace6 is described below
commit acd457ace6c6840d09768c59447954e29c259db7
Author: Mike Miller
AuthorDate: Thu Sep 22 18:35:48 2022 +
Drop FateAdmin classes (#2947)
---
server/manager/pom.xml | 4 -
.../apache/accumulo/manager/util/FateAdmin.java| 109 -
.../org/apache/accumulo/master/util/FateAdmin.java | 38 ---
3 files changed, 151 deletions(-)
diff --git a/server/manager/pom.xml b/server/manager/pom.xml
index da01ba8247..3197219820 100644
--- a/server/manager/pom.xml
+++ b/server/manager/pom.xml
@@ -31,10 +31,6 @@
Apache Accumulo Manager Server
The manager server for Apache Accumulo for load balancing and
other system-wide operations.
-
- com.beust
- jcommander
-
com.google.auto.service
auto-service
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/util/FateAdmin.java
b/server/manager/src/main/java/org/apache/accumulo/manager/util/FateAdmin.java
deleted file mode 100644
index ed76b52ec8..00
---
a/server/manager/src/main/java/org/apache/accumulo/manager/util/FateAdmin.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.manager.util;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.cli.Help;
-import org.apache.accumulo.core.conf.SiteConfiguration;
-import org.apache.accumulo.fate.AdminUtil;
-import org.apache.accumulo.fate.ReadOnlyStore;
-import org.apache.accumulo.fate.ZooStore;
-import org.apache.accumulo.fate.zookeeper.ServiceLock;
-import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
-import org.apache.accumulo.manager.Manager;
-import org.apache.accumulo.server.ServerContext;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-/**
- * A utility to administer FATE operations
- */
-public class FateAdmin {
-
- static class TxOpts {
-@Parameter(description = "...", required = true)
-List txids = new ArrayList<>();
- }
-
- @Parameters(commandDescription = "Stop an existing FATE by transaction id")
- static class FailOpts extends TxOpts {}
-
- @Parameters(commandDescription = "Delete an existing FATE by transaction id")
- static class DeleteOpts extends TxOpts {}
-
- @Parameters(commandDescription = "List the existing FATE transactions")
- static class PrintOpts {}
-
- public static void main(String[] args) throws Exception {
-Help opts = new Help();
-JCommander jc = new JCommander(opts);
-jc.setProgramName(FateAdmin.class.getName());
-LinkedHashMap txOpts = new LinkedHashMap<>(2);
-txOpts.put("fail", new FailOpts());
-txOpts.put("delete", new DeleteOpts());
-for (Entry entry : txOpts.entrySet()) {
- jc.addCommand(entry.getKey(), entry.getValue());
-}
-jc.addCommand("print", new PrintOpts());
-jc.parse(args);
-if (opts.help || jc.getParsedCommand() == null) {
- jc.usage();
- System.exit(1);
-}
-
-System.err.printf("This tool has been deprecated%nFATE administration now"
-+ " available within 'accumulo shell'%n$ fate fail ... | delete"
-+ " ... | print [...]%n%n");
-
-AdminUtil admin = new AdminUtil<>(true);
-
-try (var context = new ServerContext(SiteConfiguration.auto())) {
- final String zkRoot = context.getZooKeeperRoot();
- var zLockManagerPath = ServiceLock.path(zkRoot +
Constants.ZMANAGER_LOCK);
- var zTableLocksPath = ServiceLock.path(zkRoot + Constants.ZTABLE_LOCKS);
- String path = zkRoot + Constants.ZFATE;
- ZooRead
[accumulo] branch main updated (4205b82c63 -> 735baa1dba)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4205b82c63 Update post vote issue template for EOL releases add 735baa1dba Modify LogReader to print just the crypto params (#2935) No new revisions were added by this update. Summary of changes: .../apache/accumulo/core/crypto/CryptoTest.java| 2 +- .../org/apache/accumulo/tserver/log/DfsLogger.java | 2 +- .../apache/accumulo/tserver/logger/LogReader.java | 66 -- .../accumulo/test/functional/PerTableCryptoIT.java | 60 +++- 4 files changed, 119 insertions(+), 11 deletions(-)
[accumulo] branch main updated: Fix crypto in GenerateSplits (#2949)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 679302adbf Fix crypto in GenerateSplits (#2949)
679302adbf is described below
commit 679302adbf7cec2ec1067ef600fcd8458f94409e
Author: Mike Miller
AuthorDate: Thu Sep 22 15:01:59 2022 +
Fix crypto in GenerateSplits (#2949)
---
.../accumulo/core/file/rfile/GenerateSplits.java | 38 +-
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
index 1d33a2c8d9..df75905e43 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
@@ -38,6 +38,7 @@ import java.util.TreeSet;
import org.apache.accumulo.core.cli.ConfigOpts;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.crypto.CryptoFactoryLoader;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
@@ -45,7 +46,8 @@ import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
-import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory;
+import org.apache.accumulo.core.spi.crypto.CryptoEnvironment;
+import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.datasketches.quantiles.ItemsSketch;
@@ -83,7 +85,7 @@ public class GenerateSplits implements KeywordExecutable {
@Parameter(names = {"-sf", "--splits-file"}, description = "Output the
splits to a file")
public String outputFile;
-@Parameter(description = " { ... }")
+@Parameter(description = "[ ...] -n
| -ss ")
public List files = new ArrayList<>();
}
@@ -111,6 +113,8 @@ public class GenerateSplits implements KeywordExecutable {
Configuration hadoopConf = new Configuration();
SiteConfiguration siteConf = opts.getSiteConfiguration();
+CryptoService cryptoService = CryptoFactoryLoader
+.getServiceForClient(CryptoEnvironment.Scope.TABLE,
siteConf.getAllCryptoProperties());
boolean encode = opts.base64encode;
TreeSet splits;
@@ -140,16 +144,18 @@ public class GenerateSplits implements KeywordExecutable {
// if no size specified look at indexed keys first
if (opts.splitSize == 0) {
- splits = getIndexKeys(siteConf, hadoopConf, fs, filePaths,
requestedNumSplits, encode);
+ splits = getIndexKeys(siteConf, hadoopConf, fs, filePaths,
requestedNumSplits, encode,
+ cryptoService);
// if there weren't enough splits indexed, try again with size = 0
if (splits.size() < requestedNumSplits) {
log.info("Only found {} indexed keys but need {}. Doing a full scan on
files {}",
splits.size(), requestedNumSplits, filePaths);
-splits =
-getSplitsFromFullScan(siteConf, hadoopConf, filePaths, fs,
requestedNumSplits, encode);
+splits = getSplitsFromFullScan(siteConf, hadoopConf, filePaths, fs,
requestedNumSplits,
+encode, cryptoService);
}
} else {
- splits = getSplitsBySize(siteConf, hadoopConf, filePaths, fs, splitSize,
encode);
+ splits =
+ getSplitsBySize(siteConf, hadoopConf, filePaths, fs, splitSize,
encode, cryptoService);
}
TreeSet desiredSplits;
@@ -258,16 +264,16 @@ public class GenerateSplits implements KeywordExecutable {
* Scan the files for indexed keys first since it is more efficient than a
full file scan.
*/
private TreeSet getIndexKeys(AccumuloConfiguration accumuloConf,
Configuration hadoopConf,
- FileSystem fs, List files, int requestedNumSplits, boolean
base64encode)
- throws IOException {
+ FileSystem fs, List files, int requestedNumSplits, boolean
base64encode,
+ CryptoService cs) throws IOException {
Text[] splitArray;
List> readers = new
ArrayList<>(files.size());
List fileReaders = new ArrayList<>(files.size());
try {
for (Path file : files) {
FileSKVIterator reader =
FileOperations.getInstance().newIndexReaderBuilder()
-.forFile(file.toString(), fs, hadoopConf,
NoCryptoServiceFactory.NONE)
-.withTableConfiguration(accumuloConf).build();
+.forFile(file.toStr
[accumulo] branch main updated: Deprecate FateCommand (#2914)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new c19694c7a1 Deprecate FateCommand (#2914)
c19694c7a1 is described below
commit c19694c7a1121786017ce2e53c72f74ec0f1b324
Author: Mike Miller
AuthorDate: Wed Sep 21 12:52:36 2022 +
Deprecate FateCommand (#2914)
* Deprecate FateCommand by adding warning to usage & Shell log
* Move new code for cancel and summary options to Admin, printing message
in Shell.
* Drop pagination option for Admin
* Drop recently added tests from FateCommandTest
* Supports #2215
Co-authored-by: Dave Marion
Co-authored-by: EdColeman
---
.../java/org/apache/accumulo/fate/AdminUtil.java | 10 +-
.../java/org/apache/accumulo/fate/FateTxId.java| 8 +
.../org/apache/accumulo/server/util/Admin.java | 203 +
.../util}/fateCommand/FateSummaryReport.java | 18 +-
.../server/util}/fateCommand/FateTxnDetails.java | 6 +-
.../util}/fateCommand/SummaryReportTest.java | 2 +-
.../server/util}/fateCommand/TxnDetailsTest.java | 2 +-
.../apache/accumulo/manager/util/FateAdmin.java| 2 +-
.../accumulo/shell/commands/FateCommand.java | 121 ++--
.../accumulo/shell/commands/FateCommandTest.java | 71 ---
.../apache/accumulo/test/shell/ShellServerIT.java | 2 +-
11 files changed, 248 insertions(+), 197 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index e22d005687..80607a12f2 100644
--- a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@ -406,15 +406,15 @@ public class AdminUtil {
return (filterTxid == null) || filterTxid.isEmpty() ||
filterTxid.contains(tid);
}
- public void print(ReadOnlyTStore zs, ZooReader zk,
ServiceLock.ServiceLockPath lockPath)
- throws KeeperException, InterruptedException {
-print(zs, zk, lockPath, new Formatter(System.out), null, null);
+ public void printAll(ReadOnlyTStore zs, ZooReader zk,
+ ServiceLock.ServiceLockPath tableLocksPath) throws KeeperException,
InterruptedException {
+print(zs, zk, tableLocksPath, new Formatter(System.out), null, null);
}
- public void print(ReadOnlyTStore zs, ZooReader zk,
ServiceLock.ServiceLockPath lockPath,
+ public void print(ReadOnlyTStore zs, ZooReader zk,
ServiceLock.ServiceLockPath tableLocksPath,
Formatter fmt, Set filterTxid, EnumSet filterStatus)
throws KeeperException, InterruptedException {
-FateStatus fateStatus = getStatus(zs, zk, lockPath, filterTxid,
filterStatus);
+FateStatus fateStatus = getStatus(zs, zk, tableLocksPath, filterTxid,
filterStatus);
for (TransactionStatus txStatus : fateStatus.getTransactions()) {
fmt.format(
diff --git a/core/src/main/java/org/apache/accumulo/fate/FateTxId.java
b/core/src/main/java/org/apache/accumulo/fate/FateTxId.java
index 07b2704a16..5aff99d9bf 100644
--- a/core/src/main/java/org/apache/accumulo/fate/FateTxId.java
+++ b/core/src/main/java/org/apache/accumulo/fate/FateTxId.java
@@ -59,4 +59,12 @@ public class FateTxId {
return FastFormat.toHexString(PREFIX, tid, SUFFIX);
}
+ public static long parseTidFromUserInput(String s) {
+if (isFormatedTid(s)) {
+ return fromString(s);
+} else {
+ return Long.parseLong(s, 16);
+}
+ }
+
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 3275d3d1cd..f6605bd6f3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.server.util;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
+import static org.apache.accumulo.fate.FateTxId.parseTidFromUserInput;
import java.io.BufferedWriter;
import java.io.File;
@@ -28,12 +29,16 @@ import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Formatter;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.accumulo.core.Constants;
@@ -47,7 +52,10 @@ import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
im
[accumulo] branch main updated: Fix disabled crypto tests (#2932)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8c7a1ebe4c Fix disabled crypto tests (#2932)
8c7a1ebe4c is described below
commit 8c7a1ebe4cfe1032b2bc35c486f131293605
Author: Mike Miller
AuthorDate: Fri Sep 16 15:16:35 2022 +
Fix disabled crypto tests (#2932)
---
.../apache/accumulo/core/crypto/CryptoTest.java| 53 ++
1 file changed, 33 insertions(+), 20 deletions(-)
diff --git a/core/src/test/java/org/apache/accumulo/core/crypto/CryptoTest.java
b/core/src/test/java/org/apache/accumulo/core/crypto/CryptoTest.java
index ae613d4a27..daa530a3be 100644
--- a/core/src/test/java/org/apache/accumulo/core/crypto/CryptoTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/crypto/CryptoTest.java
@@ -20,8 +20,6 @@ package org.apache.accumulo.core.crypto;
import static com.google.common.collect.MoreCollectors.onlyElement;
import static org.apache.accumulo.core.conf.Property.INSTANCE_CRYPTO_FACTORY;
-import static org.apache.accumulo.core.conf.Property.INSTANCE_CRYPTO_PREFIX;
-import static org.apache.accumulo.core.conf.Property.TABLE_CRYPTO_PREFIX;
import static org.apache.accumulo.core.crypto.CryptoUtils.getFileDecrypter;
import static
org.apache.accumulo.core.spi.crypto.CryptoEnvironment.Scope.TABLE;
import static org.apache.accumulo.core.spi.crypto.CryptoEnvironment.Scope.WAL;
@@ -74,6 +72,7 @@ import
org.apache.accumulo.core.spi.crypto.CryptoService.CryptoException;
import org.apache.accumulo.core.spi.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.spi.crypto.FileDecrypter;
import org.apache.accumulo.core.spi.crypto.FileEncrypter;
+import org.apache.accumulo.core.spi.crypto.GenericCryptoServiceFactory;
import org.apache.accumulo.core.spi.crypto.NoCryptoService;
import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory;
import org.apache.accumulo.core.spi.crypto.PerTableCryptoServiceFactory;
@@ -118,22 +117,32 @@ public class CryptoTest {
ConfigurationCopy cfg = new
ConfigurationCopy(DefaultConfiguration.getInstance());
switch (configMode) {
case CRYPTO_TABLE_ON_DISABLED:
-cfg.set(TABLE_CRYPTO_PREFIX.getKey() + "enabled", "false");
+cfg.set(INSTANCE_CRYPTO_FACTORY,
PerTableCryptoServiceFactory.class.getName());
+cfg.set(PerTableCryptoServiceFactory.TABLE_SERVICE_NAME_PROP,
+AESCryptoService.class.getName());
+cfg.set(AESCryptoService.KEY_URI_PROPERTY,
CryptoTest.keyPath(testClass));
+cfg.set(AESCryptoService.ENCRYPT_ENABLED_PROPERTY, "false");
break;
case CRYPTO_TABLE_ON:
cfg.set(INSTANCE_CRYPTO_FACTORY,
PerTableCryptoServiceFactory.class.getName());
cfg.set(PerTableCryptoServiceFactory.TABLE_SERVICE_NAME_PROP,
AESCryptoService.class.getName());
-cfg.set(AESCryptoService.ENCRYPT_ENABLED_PROPERTY, "true");
cfg.set(AESCryptoService.KEY_URI_PROPERTY,
CryptoTest.keyPath(testClass));
+cfg.set(AESCryptoService.ENCRYPT_ENABLED_PROPERTY, "true");
break;
case CRYPTO_WAL_ON_DISABLED:
-cfg.set(INSTANCE_CRYPTO_PREFIX.getKey() + "enabled", "false");
+cfg.set(INSTANCE_CRYPTO_FACTORY,
GenericCryptoServiceFactory.class.getName());
+cfg.set(GenericCryptoServiceFactory.GENERAL_SERVICE_NAME_PROP,
+AESCryptoService.class.getName());
+cfg.set(AESCryptoService.KEY_URI_PROPERTY,
CryptoTest.keyPath(testClass));
+cfg.set(AESCryptoService.ENCRYPT_ENABLED_PROPERTY, "false");
break;
case CRYPTO_WAL_ON:
-cfg.set(INSTANCE_CRYPTO_FACTORY,
PerTableCryptoServiceFactory.class.getName());
-cfg.set(PerTableCryptoServiceFactory.WAL_NAME_PROP,
AESCryptoService.class.getName());
+cfg.set(INSTANCE_CRYPTO_FACTORY,
GenericCryptoServiceFactory.class.getName());
+cfg.set(GenericCryptoServiceFactory.GENERAL_SERVICE_NAME_PROP,
+AESCryptoService.class.getName());
cfg.set(AESCryptoService.KEY_URI_PROPERTY,
CryptoTest.keyPath(testClass));
+cfg.set(AESCryptoService.ENCRYPT_ENABLED_PROPERTY, "true");
break;
case CRYPTO_OFF:
break;
@@ -216,21 +225,23 @@ public class CryptoTest {
*/
@Test
public void testAESCryptoServiceWALDisabled() throws Exception {
-AESCryptoService cs = new AESCryptoService();
-cs.init(getAllCryptoProperties(ConfigMode.CRYPTO_WAL_ON));
+AESCryptoService csEnabled = new AESCryptoService();
+AESCryptoService csDisabled = new AESCryptoService();
+csEnabled.init(getAllCryptoProperties(ConfigMode.CRYPTO_WAL_ON));
+csDisabled.init(getAllCryptoProperties(ConfigMode.CRYPTO_WAL_ON_DISABLED));
// make sure we can read encrypted
[accumulo-testing] branch main updated: Drop security from all modules (#221)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new 6311758 Drop security from all modules (#221) 6311758 is described below commit 6311758e8e5f922d99c74a2a4d182de0b072409a Author: Mike Miller AuthorDate: Thu Sep 15 13:30:16 2022 + Drop security from all modules (#221) * Also sort modules printed by script --- bin/rwalk| 2 +- src/main/resources/randomwalk/modules/All.xml| 5 - src/main/resources/randomwalk/modules/LongClean.xml | 5 - src/main/resources/randomwalk/modules/LongDirty.xml | 5 - src/main/resources/randomwalk/modules/LongEach.xml | 4 src/main/resources/randomwalk/modules/ShortClean.xml | 5 - src/main/resources/randomwalk/modules/ShortDirty.xml | 5 - src/main/resources/randomwalk/modules/ShortEach.xml | 4 8 files changed, 1 insertion(+), 34 deletions(-) diff --git a/bin/rwalk b/bin/rwalk index 002b2e2..7a6eece 100755 --- a/bin/rwalk +++ b/bin/rwalk @@ -28,7 +28,7 @@ Usage: $0 () Modules listed below located in /src/main/resources/randomwalk/modules EOF - find "$at_home/src/main/resources/randomwalk/modules/." -maxdepth 1 -name "*.xml" -printf "%f\n" + find "$at_home/src/main/resources/randomwalk/modules/." -maxdepth 1 -name "*.xml" -printf "%f\n" | sort } export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH" diff --git a/src/main/resources/randomwalk/modules/All.xml b/src/main/resources/randomwalk/modules/All.xml index be5c815..f202635 100644 --- a/src/main/resources/randomwalk/modules/All.xml +++ b/src/main/resources/randomwalk/modules/All.xml @@ -30,7 +30,6 @@ - @@ -54,10 +53,6 @@ - - - - diff --git a/src/main/resources/randomwalk/modules/LongClean.xml b/src/main/resources/randomwalk/modules/LongClean.xml index 217afb0..757bdb7 100644 --- a/src/main/resources/randomwalk/modules/LongClean.xml +++ b/src/main/resources/randomwalk/modules/LongClean.xml @@ -29,7 +29,6 @@ - @@ -49,10 +48,6 @@ - - - - diff --git a/src/main/resources/randomwalk/modules/LongDirty.xml b/src/main/resources/randomwalk/modules/LongDirty.xml index 480e57b..49cec48 100644 --- a/src/main/resources/randomwalk/modules/LongDirty.xml +++ b/src/main/resources/randomwalk/modules/LongDirty.xml @@ -29,7 +29,6 @@ - @@ -49,10 +48,6 @@ - - - - diff --git a/src/main/resources/randomwalk/modules/LongEach.xml b/src/main/resources/randomwalk/modules/LongEach.xml index 5863341..1b4d298 100644 --- a/src/main/resources/randomwalk/modules/LongEach.xml +++ b/src/main/resources/randomwalk/modules/LongEach.xml @@ -36,10 +36,6 @@ - - - - diff --git a/src/main/resources/randomwalk/modules/ShortClean.xml b/src/main/resources/randomwalk/modules/ShortClean.xml index 19bb807..8562171 100644 --- a/src/main/resources/randomwalk/modules/ShortClean.xml +++ b/src/main/resources/randomwalk/modules/ShortClean.xml @@ -29,7 +29,6 @@ - @@ -49,10 +48,6 @@ - - - - diff --git a/src/main/resources/randomwalk/modules/ShortDirty.xml b/src/main/resources/randomwalk/modules/ShortDirty.xml index 73fd988..ff2846b 100644 --- a/src/main/resources/randomwalk/modules/ShortDirty.xml +++ b/src/main/resources/randomwalk/modules/ShortDirty.xml @@ -29,7 +29,6 @@ - @@ -49,10 +48,6 @@ - - - - diff --git a/src/main/resources/randomwalk/modules/ShortEach.xml b/src/main/resources/randomwalk/modules/ShortEach.xml index fe7c857..5da6880 100644 --- a/src/main/resources/randomwalk/modules/ShortEach.xml +++ b/src/main/resources/randomwalk/modules/ShortEach.xml @@ -36,10 +36,6 @@ - - - -
[accumulo] branch main updated (4cc8e3bbe1 -> 98030c3789)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4cc8e3bbe1 ScanServerSelector - Break nested interfaces into their own files (#2896) add 98030c3789 Add new params to import table (#2879) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/core/Constants.java | 1 + .../core/client/admin/ImportConfiguration.java | 100 ++ .../core/client/admin/TableOperations.java | 7 +- .../core/clientImpl/ImportConfigurationImpl.java | 64 + .../core/clientImpl/TableOperationsImpl.java | 19 +-- .../core/client/admin/ImportConfigurationTest.java | 55 .../core/clientImpl/TableOperationsHelperTest.java | 3 +- .../accumulo/manager/FateServiceHandler.java | 13 +- .../tableOps/tableImport/FinishImportTable.java| 11 +- .../manager/tableOps/tableImport/ImportTable.java | 8 +- .../tableOps/tableImport/ImportedTableInfo.java| 2 + .../tableOps/tableImport/MapImportFileNames.java | 3 +- .../tableImport/PopulateMetadataTable.java | 6 +- .../shell/commands/ImportTableCommand.java | 25 +++- .../org/apache/accumulo/test/AuditMessageIT.java | 3 +- .../org/apache/accumulo/test/ImportExportIT.java | 153 - .../org/apache/accumulo/test/NamespacesIT.java | 5 +- 17 files changed, 448 insertions(+), 30 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/client/admin/ImportConfiguration.java create mode 100644 core/src/main/java/org/apache/accumulo/core/clientImpl/ImportConfigurationImpl.java create mode 100644 core/src/test/java/org/apache/accumulo/core/client/admin/ImportConfigurationTest.java
[accumulo] branch main updated: Fix Fate enum name (#2904)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 114704bb58 Fix Fate enum name (#2904)
114704bb58 is described below
commit 114704bb58a37073a0401c3b6f479761089bd1fb
Author: Mike Miller
AuthorDate: Thu Sep 1 16:08:05 2022 +
Fix Fate enum name (#2904)
* Rename enum to TX_NAME and pass in name to Fate seedTransaction()
* Clean up FateLogger log and fateCommand print
---
.../apache/accumulo/core/logging/FateLogger.java | 2 +-
.../java/org/apache/accumulo/fate/AdminUtil.java | 24 +++---
.../main/java/org/apache/accumulo/fate/Fate.java | 7 ++--
.../accumulo/manager/FateServiceHandler.java | 38 +++---
.../manager/ManagerClientServiceHandler.java | 4 +--
.../manager/metrics/fate/FateMetricValues.java | 2 +-
.../commands/fateCommand/FateSummaryReport.java| 4 +--
.../shell/commands/fateCommand/FateTxnDetails.java | 8 ++---
.../commands/fateCommand/SummaryReportTest.java| 2 +-
.../shell/commands/fateCommand/TxnDetailsTest.java | 6 ++--
.../accumulo/test/fate/zookeeper/FateIT.java | 10 +++---
.../test/functional/FateConcurrencyIT.java | 7 ++--
12 files changed, 56 insertions(+), 58 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
index 56a9790205..ff630d7906 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
@@ -133,7 +133,7 @@ public class FateLogger {
public void setTransactionInfo(long tid, Fate.TxInfo txInfo,
Serializable val) {
store.setTransactionInfo(tid, txInfo, val);
if (storeLog.isTraceEnabled())
- storeLog.trace("{} setting {} txInfo to {}", formatTid(tid), txInfo,
val);
+ storeLog.trace("{} setting {} to {}", formatTid(tid), txInfo, val);
}
@Override
diff --git a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index dbe1fa5def..e22d005687 100644
--- a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@ -73,18 +73,18 @@ public class AdminUtil {
private final long txid;
private final TStatus status;
-private final String repoTarget;
+private final String txName;
private final List hlocks;
private final List wlocks;
private final String top;
private final long timeCreated;
-private TransactionStatus(Long tid, TStatus status, String repoTarget,
List hlocks,
+private TransactionStatus(Long tid, TStatus status, String txName,
List hlocks,
List wlocks, String top, Long timeCreated) {
this.txid = tid;
this.status = status;
- this.repoTarget = repoTarget;
+ this.txName = txName;
this.hlocks = Collections.unmodifiableList(hlocks);
this.wlocks = Collections.unmodifiableList(wlocks);
this.top = top;
@@ -105,10 +105,10 @@ public class AdminUtil {
}
/**
- * @return The repo target for the operation on the top of the stack for
this Fate operation.
+ * @return The name of the transaction running.
*/
-public String getRepoTarget() {
- return repoTarget;
+public String getTxName() {
+ return txName;
}
/**
@@ -364,7 +364,7 @@ public class AdminUtil {
zs.reserve(tid);
- String repoTarget = (String) zs.getTransactionInfo(tid,
Fate.TxInfo.REPO_TARGET);
+ String txName = (String) zs.getTransactionInfo(tid, Fate.TxInfo.TX_NAME);
List hlocks = heldLocks.remove(tid);
@@ -390,8 +390,7 @@ public class AdminUtil {
zs.unreserve(tid, 0);
if (includeByStatus(status, filterStatus) && includeByTxid(tid,
filterTxid)) {
-statuses
-.add(new TransactionStatus(tid, status, repoTarget, hlocks,
wlocks, top, timeCreated));
+statuses.add(new TransactionStatus(tid, status, txName, hlocks,
wlocks, top, timeCreated));
}
}
@@ -419,10 +418,9 @@ public class AdminUtil {
for (TransactionStatus txStatus : fateStatus.getTransactions()) {
fmt.format(
- "txid: %s status: %-18s op: %-15s locked: %-15s locking: %-15s
top: %-15s created: %s%n",
- txStatus.getTxid(), txStatus.getStatus(), txStatus.getRepoTarget(),
- txStatus.getHeldLocks(), txStatus.getWaitingLocks(),
txStatus.getTop(),
- txStatus.getTimeCreatedFormatted());
+ "%-15s txid: %s status: %-18s locked: %-15s locking: %-15s op:
%-15s created: %s%n",
+ txStatus.getTxName(), txStatus.getTxid
[accumulo] branch main updated (b454e0c1e4 -> 202c765d65)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b454e0c1e4 Improvements to AESCryptoService (#2895) add 202c765d65 Make Text field transient in BadLocationStateException (#2897) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/core/metadata/TabletLocationState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] branch main updated: Improvements to AESCryptoService (#2895)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new b454e0c1e4 Improvements to AESCryptoService (#2895)
b454e0c1e4 is described below
commit b454e0c1e4ff0bde55ad9e7a6d9ced517d1c3943
Author: Mike Miller
AuthorDate: Fri Aug 26 13:47:55 2022 +
Improvements to AESCryptoService (#2895)
* Reduce Cipher.getInstance calls to once per file vs once per block
* Move wrap cipher to ThreadLocal objects
* Add logger and init checks
---
.../accumulo/core/spi/crypto/AESCryptoService.java | 148 ++---
.../apache/accumulo/core/crypto/CryptoTest.java| 12 +-
2 files changed, 108 insertions(+), 52 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/crypto/AESCryptoService.java
b/core/src/main/java/org/apache/accumulo/core/spi/crypto/AESCryptoService.java
index cfe8484b33..bc223f5557 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/crypto/AESCryptoService.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/crypto/AESCryptoService.java
@@ -55,6 +55,8 @@ import
org.apache.accumulo.core.crypto.streams.BlockedOutputStream;
import org.apache.accumulo.core.crypto.streams.DiscardCloseOutputStream;
import org.apache.accumulo.core.crypto.streams.RFileCipherOutputStream;
import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -62,6 +64,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* Example implementation of AES encryption for Accumulo
*/
public class AESCryptoService implements CryptoService {
+ private static final Logger log =
LoggerFactory.getLogger(AESCryptoService.class);
+ private volatile boolean initialized = false;
+
// properties required for using this service
private static final String CRYPTO_PREFIX = "instance.crypto.opts.";
private static final String KEY_URI = CRYPTO_PREFIX + "key.uri";
@@ -85,8 +90,35 @@ public class AESCryptoService implements CryptoService {
private static final FileEncrypter DISABLED = new NoFileEncrypter();
+ private static final ThreadLocal KEY_WRAP_CIPHER = new
ThreadLocal() {
+@SuppressFBWarnings(value = "CIPHER_INTEGRITY",
+justification = "integrity not needed for key wrap")
+@Override
+protected Cipher initialValue() {
+ try {
+return Cipher.getInstance(KEY_WRAP_TRANSFORM);
+ } catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
+throw new CryptoException("Error creating Cipher for AESWrap", e);
+ }
+}
+ };
+
+ private static final ThreadLocal KEY_UNWRAP_CIPHER = new
ThreadLocal() {
+@SuppressFBWarnings(value = "CIPHER_INTEGRITY",
+justification = "integrity not needed for key wrap")
+@Override
+protected Cipher initialValue() {
+ try {
+return Cipher.getInstance(KEY_WRAP_TRANSFORM);
+ } catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
+throw new CryptoException("Error creating Cipher for AESWrap", e);
+ }
+}
+ };
+
@Override
public void init(Map conf) throws CryptoException {
+ensureNotInit();
String keyLocation =
Objects.requireNonNull(conf.get(KEY_URI), "Config property " + KEY_URI
+ " is required.");
String enabledProp = conf.get(ENCRYPT_ENABLED);
@@ -107,10 +139,13 @@ public class AESCryptoService implements CryptoService {
}
Objects.requireNonNull(this.encryptingKek,
"Encrypting Key Encryption Key was null, init failed");
+log.debug("Successfully initialized crypto Key from {}", keyLocation);
+initialized = true;
}
@Override
public FileEncrypter getFileEncrypter(CryptoEnvironment environment) {
+ensureInit();
if (!encryptEnabled) {
return DISABLED;
}
@@ -131,6 +166,7 @@ public class AESCryptoService implements CryptoService {
@Override
public FileDecrypter getFileDecrypter(CryptoEnvironment environment) {
+ensureInit();
CryptoModule cm;
byte[] decryptionParams = environment.getDecryptionParams();
if (decryptionParams == null || checkNoCrypto(decryptionParams))
@@ -205,7 +241,6 @@ public class AESCryptoService implements CryptoService {
private static byte[] createCryptoParameters(String version, Key
encryptingKek,
String encryptingKekId, String encryptingKeyManager, Key fek) {
-byte[] bytes;
try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream params = new DataOutputStream(baos)) {
params.writeUTF(AESCryptoService.class.getName());
@@ -216,11 +251,11 @@ public class AESCryptoService implements Crypt
[accumulo] branch main updated: Removing LocalityCheck utility (#2808)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 285bfb7924 Removing LocalityCheck utility (#2808)
285bfb7924 is described below
commit 285bfb792422c968df33a95a6a81b4739851607b
Author: Christopher L. Shannon
AuthorDate: Thu Aug 25 08:13:14 2022 -0400
Removing LocalityCheck utility (#2808)
* Utility is no longer useful so it can be removed.
* This closes #2807
---
.../apache/accumulo/server/util/LocalityCheck.java | 129 -
1 file changed, 129 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
deleted file mode 100644
index 77a4d67cba..00
---
a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.util;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.Accumulo;
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.ValidationUtil;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
-import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.core.util.HostAndPort;
-import org.apache.accumulo.server.cli.ServerUtilOpts;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import io.opentelemetry.api.trace.Span;
-import io.opentelemetry.context.Scope;
-
-public class LocalityCheck {
-
- public int run(String[] args) throws Exception {
-ServerUtilOpts opts = new ServerUtilOpts();
-opts.parseArgs(LocalityCheck.class.getName(), args);
-
-Span span = TraceUtil.startSpan(LocalityCheck.class, "run");
-try (Scope scope = span.makeCurrent()) {
-
- VolumeManager fs = opts.getServerContext().getVolumeManager();
- try (AccumuloClient accumuloClient =
- Accumulo.newClient().from(opts.getClientProps()).build()) {
-Scanner scanner = accumuloClient.createScanner(MetadataTable.NAME,
Authorizations.EMPTY);
-scanner.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
-scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
-scanner.setRange(TabletsSection.getRange());
-
-Map totalBlocks = new HashMap<>();
-Map localBlocks = new HashMap<>();
-ArrayList files = new ArrayList<>();
-
-for (Entry entry : scanner) {
- Key key = entry.getKey();
- if (key.compareColumnFamily(CurrentLocationColumnFamily.NAME) == 0) {
-String location = entry.getValue().toString();
-String[] parts = location.split(":");
-String host = parts[0];
-addBlocks(fs, host, files, totalBlocks, localBlocks);
-files.clear();
- } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
-
files.add(ValidationUtil.validate(key.getColumnQualifierData().toString()));
- }
-}
-System.out.println(" Server %local total blocks");
-for (Entry entry : totalBlocks.entrySet()) {
- final String host = entry.getKey();
- final Long blocksForHost = entry.getValue();
-
[accumulo] branch main updated: Cleanup Accumulo Admin utilities (#2881)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ab638b796e Cleanup Accumulo Admin utilities (#2881)
ab638b796e is described below
commit ab638b796e7080ee2b8db4cf94ef22f4126e566b
Author: Christopher L. Shannon
AuthorDate: Tue Aug 23 12:31:16 2022 -0400
Cleanup Accumulo Admin utilities (#2881)
* Remove main methods from utilities now that they are exposed through
KeywordExecutable and the admin command.
* Also sort the commands in the help section alphabetically.
* Issue #2807
---
.../org/apache/accumulo/server/util/Admin.java | 54 +++---
.../apache/accumulo/server/util/ChangeSecret.java | 13 +-
.../accumulo/server/util/DeleteZooInstance.java| 20 +---
.../accumulo/server/util/RestoreZookeeper.java | 18 +---
.../accumulo/server/util/TabletServerLocks.java| 33 -
.../server/util/VerifyTabletAssignments.java | 24 +-
6 files changed, 41 insertions(+), 121 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index fe3bddccfe..3275d3d1cd 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -226,49 +226,52 @@ public class Admin implements KeywordExecutable {
JCommander cl = new JCommander(opts);
cl.setProgramName("accumulo admin");
-CheckTabletsCommand checkTabletsCommand = new CheckTabletsCommand();
-cl.addCommand("checkTablets", checkTabletsCommand);
-
ChangeSecretCommand changeSecretCommand = new ChangeSecretCommand();
cl.addCommand("changeSecret", changeSecretCommand);
+CheckTabletsCommand checkTabletsCommand = new CheckTabletsCommand();
+cl.addCommand("checkTablets", checkTabletsCommand);
+
DeleteZooInstanceCommand deleteZooInstanceOpts = new
DeleteZooInstanceCommand();
cl.addCommand("deleteZooInstance", deleteZooInstanceOpts);
-RestoreZooCommand restoreZooOpts = new RestoreZooCommand();
-cl.addCommand("restoreZoo", restoreZooOpts);
+DumpConfigCommand dumpConfigCommand = new DumpConfigCommand();
+cl.addCommand("dumpConfig", dumpConfigCommand);
+
+ListInstancesCommand listInstancesOpts = new ListInstancesCommand();
+cl.addCommand("listInstances", listInstancesOpts);
-ListInstancesCommand listIntancesOpts = new ListInstancesCommand();
-cl.addCommand("listInstances", listIntancesOpts);
+TabletServerLocksCommand tServerLocksOpts = new TabletServerLocksCommand();
+cl.addCommand("locks", tServerLocksOpts);
PingCommand pingCommand = new PingCommand();
cl.addCommand("ping", pingCommand);
-DumpConfigCommand dumpConfigCommand = new DumpConfigCommand();
-cl.addCommand("dumpConfig", dumpConfigCommand);
+RestoreZooCommand restoreZooOpts = new RestoreZooCommand();
+cl.addCommand("restoreZoo", restoreZooOpts);
-VolumesCommand volumesCommand = new VolumesCommand();
-cl.addCommand("volumes", volumesCommand);
+RandomizeVolumesCommand randomizeVolumesOpts = new
RandomizeVolumesCommand();
+cl.addCommand("randomizeVolumes", randomizeVolumesOpts);
StopCommand stopOpts = new StopCommand();
cl.addCommand("stop", stopOpts);
+
StopAllCommand stopAllOpts = new StopAllCommand();
cl.addCommand("stopAll", stopAllOpts);
+
StopManagerCommand stopManagerOpts = new StopManagerCommand();
cl.addCommand("stopManager", stopManagerOpts);
+
StopMasterCommand stopMasterOpts = new StopMasterCommand();
cl.addCommand("stopMaster", stopMasterOpts);
-RandomizeVolumesCommand randomizeVolumesOpts = new
RandomizeVolumesCommand();
-cl.addCommand("randomizeVolumes", randomizeVolumesOpts);
-
-TabletServerLocksCommand tServerLocksOpts = new TabletServerLocksCommand();
-cl.addCommand("locks", tServerLocksOpts);
-
VerifyTabletAssignmentsCommand verifyTabletAssignmentsOpts =
new VerifyTabletAssignmentsCommand();
cl.addCommand("verifyTabletAssigns", verifyTabletAssignmentsOpts);
+VolumesCommand volumesCommand = new VolumesCommand();
+cl.addCommand("volumes", volumesCommand);
+
cl.parse(args);
if (opts.help || cl.getParsedCommand() == null) {
@@ -289,8 +292,8 @@ public class Admin implements KeywordExecutable {
int rc = 0;
if (cl.getParsedCommand().equals("listInstances")) {
-ListInstances.listInstances(context.getZooKeepers(),
listIntancesO
[accumulo] branch main updated: Rename ZooStore methods and change Strings to enum (#2842)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e08287065d Rename ZooStore methods and change Strings to enum (#2842)
e08287065d is described below
commit e08287065d3af4773dd97a3991686abc3002f6dd
Author: Christopher L. Shannon
AuthorDate: Fri Aug 12 15:53:19 2022 -0400
Rename ZooStore methods and change Strings to enum (#2842)
* This closes #2841
---
.../apache/accumulo/core/logging/FateLogger.java | 11 +-
.../java/org/apache/accumulo/fate/AdminUtil.java | 24 +++--
.../java/org/apache/accumulo/fate/AgeOffStore.java | 8 +++
.../main/java/org/apache/accumulo/fate/Fate.java | 25 +++---
.../org/apache/accumulo/fate/ReadOnlyRepo.java | 2 +-
.../org/apache/accumulo/fate/ReadOnlyStore.java| 8 +++
.../org/apache/accumulo/fate/ReadOnlyTStore.java | 8 +++
.../main/java/org/apache/accumulo/fate/TStore.java | 12 ++-
.../java/org/apache/accumulo/fate/ZooStore.java| 12 +--
.../apache/accumulo/fate/ReadOnlyStoreTest.java| 9
.../manager/metrics/fate/FateMetricValues.java | 2 +-
.../accumulo/manager/tableOps/ManagerRepo.java | 2 +-
.../accumulo/manager/tableOps/TraceRepo.java | 10 -
.../commands/fateCommand/FateSummaryReport.java| 2 +-
.../shell/commands/fateCommand/FateTxnDetails.java | 4 ++--
.../commands/fateCommand/SummaryReportTest.java| 2 +-
.../shell/commands/fateCommand/TxnDetailsTest.java | 6 +++---
.../test/functional/FateConcurrencyIT.java | 4 ++--
18 files changed, 82 insertions(+), 69 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
index 2b913d59aa..56a9790205 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
@@ -25,6 +25,7 @@ import java.util.EnumSet;
import java.util.List;
import java.util.function.Function;
+import org.apache.accumulo.fate.Fate;
import org.apache.accumulo.fate.ReadOnlyRepo;
import org.apache.accumulo.fate.Repo;
import org.apache.accumulo.fate.StackOverflowException;
@@ -80,8 +81,8 @@ public class FateLogger {
}
@Override
- public Serializable getProperty(long tid, String prop) {
-return store.getProperty(tid, prop);
+ public Serializable getTransactionInfo(long tid, Fate.TxInfo txInfo) {
+return store.getTransactionInfo(tid, txInfo);
}
@Override
@@ -129,10 +130,10 @@ public class FateLogger {
}
@Override
- public void setProperty(long tid, String prop, Serializable val) {
-store.setProperty(tid, prop, val);
+ public void setTransactionInfo(long tid, Fate.TxInfo txInfo,
Serializable val) {
+store.setTransactionInfo(tid, txInfo, val);
if (storeLog.isTraceEnabled())
- storeLog.trace("{} setting {} node data to {}", formatTid(tid),
prop, val);
+ storeLog.trace("{} setting {} txInfo to {}", formatTid(tid), txInfo,
val);
}
@Override
diff --git a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index bf1ceef4e6..dbe1fa5def 100644
--- a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@ -73,18 +73,18 @@ public class AdminUtil {
private final long txid;
private final TStatus status;
-private final String debug;
+private final String repoTarget;
private final List hlocks;
private final List wlocks;
private final String top;
private final long timeCreated;
-private TransactionStatus(Long tid, TStatus status, String debug,
List hlocks,
+private TransactionStatus(Long tid, TStatus status, String repoTarget,
List hlocks,
List wlocks, String top, Long timeCreated) {
this.txid = tid;
this.status = status;
- this.debug = debug;
+ this.repoTarget = repoTarget;
this.hlocks = Collections.unmodifiableList(hlocks);
this.wlocks = Collections.unmodifiableList(wlocks);
this.top = top;
@@ -105,10 +105,10 @@ public class AdminUtil {
}
/**
- * @return The debug info for the operation on the top of the stack for
this Fate operation.
+ * @return The repo target for the operation on the top of the stack for
this Fate operation.
*/
-public String getDebug() {
- return debug;
+public String getRepoTarget() {
+ return repoTarget;
}
/**
@@ -364,7 +364,7 @@ public class AdminUtil {
zs.reserve(tid);
- String debug = (String
[accumulo] branch main updated: Fast fail table export if online check passes (#2877)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 47cdd872d5 Fast fail table export if online check passes (#2877)
47cdd872d5 is described below
commit 47cdd872d5ba6a3cdfb47cf15b6fadcd4bfc6fe9
Author: Mike Miller
AuthorDate: Fri Aug 12 19:16:11 2022 +
Fast fail table export if online check passes (#2877)
---
.../java/org/apache/accumulo/core/client/admin/TableOperations.java | 3 ++-
.../org/apache/accumulo/core/clientImpl/TableOperationsImpl.java | 5 +
test/src/main/java/org/apache/accumulo/test/ImportExportIT.java | 5 +
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 09a1b0e3dd..e4bf572eb5 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -187,7 +187,8 @@ public interface TableOperations {
}
/**
- * Imports a table exported via exportTable and copied via hadoop distcp.
+ * Imports a table exported via {@link #exportTable(String, String)} and
then copied via hadoop
+ * distcp.
*
* @param tableName
* Name of a table to create and import into.
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index e2cfc4840c..849465f62f 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -1621,6 +1621,11 @@ public class TableOperationsImpl extends
TableOperationsHelper {
EXISTING_TABLE_NAME.validate(tableName);
checkArgument(exportDir != null, "exportDir is null");
+if (isOnline(tableName)) {
+ throw new IllegalStateException("The table " + tableName + " is online;
exportTable requires"
+ + " a table to be offline before exporting.");
+}
+
List args =
Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8)),
ByteBuffer.wrap(exportDir.getBytes(UTF_8)));
Map opts = Collections.emptyMap();
diff --git a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
index 429fd1bc44..fc481fc655 100644
--- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -122,6 +123,10 @@ public class ImportExportIT extends AccumuloClusterHarness
{
log.info("Exporting table to {}", exportDir);
log.info("Importing table from {}", importDirs);
+ // test fast fail offline check
+ assertThrows(IllegalStateException.class,
+ () -> client.tableOperations().exportTable(srcTable,
exportDir.toString()));
+
// Offline the table
client.tableOperations().offline(srcTable, true);
// Then export it
[accumulo] branch main updated: Adding TabletServerLocks utility to accumulo admin command (#2817)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new abc89d69a0 Adding TabletServerLocks utility to accumulo admin command
(#2817)
abc89d69a0 is described below
commit abc89d69a0d620065442f84a273edfcf6bff84e1
Author: Christopher L. Shannon
AuthorDate: Fri Aug 12 10:20:39 2022 -0400
Adding TabletServerLocks utility to accumulo admin command (#2817)
Issue #2807
---
.../org/apache/accumulo/server/util/Admin.java | 13 +
.../accumulo/server/util/TabletServerLocks.java| 59 --
2 files changed, 46 insertions(+), 26 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 002eb9fbe0..fe3bddccfe 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -176,6 +176,13 @@ public class Admin implements KeywordExecutable {
commandDescription = "Changes the unique secret given to the instance
that all servers must know.")
static class ChangeSecretCommand {}
+ @Parameters(
+ commandDescription = "List or delete Tablet Server locks. Default with
no arguments is to list the locks.")
+ static class TabletServerLocksCommand {
+@Parameter(names = "-delete", description = "specify a tablet server lock
to delete")
+String delete = null;
+ }
+
@Parameters(commandDescription = "Deletes instance name or id from
zookeeper.")
static class DeleteZooInstanceCommand {
@Parameter(names = {"-i", "--instance"}, description = "the instance name
or id to delete")
@@ -255,6 +262,9 @@ public class Admin implements KeywordExecutable {
RandomizeVolumesCommand randomizeVolumesOpts = new
RandomizeVolumesCommand();
cl.addCommand("randomizeVolumes", randomizeVolumesOpts);
+TabletServerLocksCommand tServerLocksOpts = new TabletServerLocksCommand();
+cl.addCommand("locks", tServerLocksOpts);
+
VerifyTabletAssignmentsCommand verifyTabletAssignmentsOpts =
new VerifyTabletAssignmentsCommand();
cl.addCommand("verifyTabletAssigns", verifyTabletAssignmentsOpts);
@@ -316,6 +326,9 @@ public class Admin implements KeywordExecutable {
DeleteZooInstance.deleteZooInstance(deleteZooInstanceOpts.instance);
} else if (cl.getParsedCommand().equals("restoreZoo")) {
RestoreZookeeper.restoreZookeeper(conf, restoreZooOpts.file,
restoreZooOpts.overwrite);
+ } else if (cl.getParsedCommand().equals("locks")) {
+TabletServerLocks.tabletServerLocks("accumulo locks", context,
+args.length > 2 ? args[2] : null, tServerLocksOpts.delete);
} else {
everything = cl.getParsedCommand().equals("stopAll");
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
index fff7dc5af5..51c9e4658d 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/TabletServerLocks.java
@@ -35,44 +35,51 @@ import com.beust.jcommander.Parameter;
public class TabletServerLocks {
static class Opts extends Help {
-@Parameter(names = "-list")
-boolean list = false;
@Parameter(names = "-delete")
String delete = null;
}
- public static void main(String[] args) throws Exception {
-
-try (var context = new ServerContext(SiteConfiguration.auto())) {
- String tserverPath = context.getZooKeeperRoot() + Constants.ZTSERVERS;
- Opts opts = new Opts();
- opts.parseArgs(TabletServerLocks.class.getName(), args);
+ public static void tabletServerLocks(String commandName, final ServerContext
context,
+ final String lock, final String delete) throws Exception {
- ZooCache cache = context.getZooCache();
- ZooReaderWriter zoo = context.getZooReaderWriter();
+String tserverPath = context.getZooKeeperRoot() + Constants.ZTSERVERS;
- if (opts.list) {
+ZooCache cache = context.getZooCache();
+ZooReaderWriter zoo = context.getZooReaderWriter();
-List tabletServers = zoo.getChildren(tserverPath);
+if (delete == null) {
+ List tabletServers = zoo.getChildren(tserverPath);
-for (String tabletServer : tabletServers) {
- var zLockPath = ServiceLock.path(tserverPath + "/" + tabletServer);
- byte[] lockData = ServiceLock.getLockData(cache, zLockPath, null);
- String holder = null;
-
[accumulo] branch main updated: Adding RestoreZookeeper utility to accumulo admin command (#2816)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ed69b0d26d Adding RestoreZookeeper utility to accumulo admin command
(#2816)
ed69b0d26d is described below
commit ed69b0d26d9bdbabaa3aaddb26c308c8213be3f1
Author: Christopher L. Shannon
AuthorDate: Fri Aug 12 09:28:56 2022 -0400
Adding RestoreZookeeper utility to accumulo admin command (#2816)
Issue #2807
---
.../accumulo/fate/zookeeper/ZooReaderWriter.java| 4 ++--
.../java/org/apache/accumulo/server/util/Admin.java | 14 ++
.../accumulo/server/util/RestoreZookeeper.java | 21 +
3 files changed, 29 insertions(+), 10 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
b/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
index 4637034fce..9a86bc0fac 100644
--- a/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
+++ b/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
@@ -24,8 +24,8 @@ import static java.util.Objects.requireNonNull;
import java.util.List;
import
org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.zookeeper.CreateMode;
@@ -40,7 +40,7 @@ public class ZooReaderWriter extends ZooReader {
byte[] mutate(byte[] currentValue) throws
AcceptableThriftTableOperationException;
}
- public ZooReaderWriter(SiteConfiguration conf) {
+ public ZooReaderWriter(AccumuloConfiguration conf) {
this(conf.get(Property.INSTANCE_ZK_HOST),
(int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
conf.get(Property.INSTANCE_SECRET));
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 168530b9ee..002eb9fbe0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -182,6 +182,15 @@ public class Admin implements KeywordExecutable {
String instance;
}
+ @Parameters(commandDescription = "Restore Zookeeper data from a file.")
+ static class RestoreZooCommand {
+@Parameter(names = "--overwrite")
+boolean overwrite = false;
+
+@Parameter(names = "--file")
+String file;
+ }
+
public static void main(String[] args) {
new Admin().execute(args);
}
@@ -219,6 +228,9 @@ public class Admin implements KeywordExecutable {
DeleteZooInstanceCommand deleteZooInstanceOpts = new
DeleteZooInstanceCommand();
cl.addCommand("deleteZooInstance", deleteZooInstanceOpts);
+RestoreZooCommand restoreZooOpts = new RestoreZooCommand();
+cl.addCommand("restoreZoo", restoreZooOpts);
+
ListInstancesCommand listIntancesOpts = new ListInstancesCommand();
cl.addCommand("listInstances", listIntancesOpts);
@@ -302,6 +314,8 @@ public class Admin implements KeywordExecutable {
ChangeSecret.changeSecret(context, conf);
} else if (cl.getParsedCommand().equals("deleteZooInstance")) {
DeleteZooInstance.deleteZooInstance(deleteZooInstanceOpts.instance);
+ } else if (cl.getParsedCommand().equals("restoreZoo")) {
+RestoreZookeeper.restoreZookeeper(conf, restoreZooOpts.file,
restoreZooOpts.overwrite);
} else {
everything = cl.getParsedCommand().equals("stopAll");
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
index 319c59c322..51016448ab 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java
@@ -29,6 +29,7 @@ import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
@@ -116,15 +117,13 @@ public class RestoreZookeeper {
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
justification = "code runs in same security context as user who
[accumulo] branch main updated: Add new CloneConfiguration API method (#2874)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e4299fa5b8 Add new CloneConfiguration API method (#2874)
e4299fa5b8 is described below
commit e4299fa5b82f7fcd835b5e81669882d90aa99ebd
Author: Mike Miller
AuthorDate: Fri Aug 12 12:43:31 2022 +
Add new CloneConfiguration API method (#2874)
* Add new convenience method for empty object
* Create private methods for reuse in TableOperationsImpl
* Some various code clean up
---
.../core/client/admin/CloneConfiguration.java | 30 +--
.../core/clientImpl/CloneConfigurationImpl.java| 7 ++--
.../core/clientImpl/TableOperationsImpl.java | 43 +-
.../accumulo/manager/FateServiceHandler.java | 6 +--
.../accumulo/test/functional/CloneTestIT.java | 8 ++--
5 files changed, 55 insertions(+), 39 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/CloneConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/CloneConfiguration.java
index fa2c104258..a47fa7a58a 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/CloneConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/CloneConfiguration.java
@@ -35,14 +35,14 @@ public interface CloneConfiguration {
*
* @return true if memory is flushed in the source table before cloning.
*/
- public boolean isFlush();
+ boolean isFlush();
/**
* The source table properties are copied. This allows overriding of some of
those properties.
*
* @return The source table properties to override.
*/
- public Map getPropertiesToSet();
+ Map getPropertiesToSet();
/**
* The source table properties are copied, this allows reverting to system
defaults for some of
@@ -50,7 +50,7 @@ public interface CloneConfiguration {
*
* @return The properties that are to be reverted to system defaults.
*/
- public Set getPropertiesToExclude();
+ Set getPropertiesToExclude();
/**
* The new table is normally brought online after the cloning process. This
allows leaving the new
@@ -58,21 +58,21 @@ public interface CloneConfiguration {
*
* @return true if the new table is to be kept offline after cloning.
*/
- public boolean isKeepOffline();
+ boolean isKeepOffline();
/**
* A CloneConfiguration builder
*
* @since 1.10 and 2.1
*/
- public static interface Builder {
+ interface Builder {
/**
* Determines if memory is flushed in the source table before cloning.
*
* @param flush
* true if memory is flushed in the source table before cloning.
*/
-public Builder setFlush(boolean flush);
+Builder setFlush(boolean flush);
/**
* The source table properties are copied. This allows overriding of some
of those properties.
@@ -80,7 +80,7 @@ public interface CloneConfiguration {
* @param propertiesToSet
* The source table properties to override.
*/
-public Builder setPropertiesToSet(Map propertiesToSet);
+Builder setPropertiesToSet(Map propertiesToSet);
/**
* The source table properties are copied, this allows reverting to system
defaults for some of
@@ -89,7 +89,7 @@ public interface CloneConfiguration {
* @param propertiesToExclude
* The properties that are to be reverted to system defaults.
*/
-public Builder setPropertiesToExclude(Set propertiesToExclude);
+Builder setPropertiesToExclude(Set propertiesToExclude);
/**
* The new table is normally brought online after the cloning process.
This allows leaving the
@@ -98,21 +98,29 @@ public interface CloneConfiguration {
* @param keepOffline
* true if the new table is to be kept offline after cloning.
*/
-public Builder setKeepOffline(boolean keepOffline);
+Builder setKeepOffline(boolean keepOffline);
/**
* Build the clone configuration
*
* @return the built immutable clone configuration
*/
-public CloneConfiguration build();
+CloneConfiguration build();
}
/**
* @return a {@link CloneConfiguration} builder
*/
- public static CloneConfiguration.Builder builder() {
+ static CloneConfiguration.Builder builder() {
return new CloneConfigurationImpl();
}
+ /**
+ * @return an empty configuration object with the default settings.
+ * @since 2.1.0
+ */
+ static CloneConfiguration empty() {
+return builder().build();
+ }
+
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java
index d380090dae..9549fd6c5d 100644
---
a/core/src/main/java/org
[accumulo] branch main updated: Adding DeleteZooInstance utility to accumulo admin command (#2815)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 469fe9f681 Adding DeleteZooInstance utility to accumulo admin command
(#2815)
469fe9f681 is described below
commit 469fe9f6815912507bc2e3d5c6d69d38475a67d9
Author: Christopher L. Shannon
AuthorDate: Fri Aug 12 08:24:24 2022 -0400
Adding DeleteZooInstance utility to accumulo admin command (#2815)
Issue #2807
---
.../org/apache/accumulo/server/util/Admin.java | 11 +++
.../accumulo/server/util/DeleteZooInstance.java| 34 +-
2 files changed, 31 insertions(+), 14 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 053e0c197b..168530b9ee 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -176,6 +176,12 @@ public class Admin implements KeywordExecutable {
commandDescription = "Changes the unique secret given to the instance
that all servers must know.")
static class ChangeSecretCommand {}
+ @Parameters(commandDescription = "Deletes instance name or id from
zookeeper.")
+ static class DeleteZooInstanceCommand {
+@Parameter(names = {"-i", "--instance"}, description = "the instance name
or id to delete")
+String instance;
+ }
+
public static void main(String[] args) {
new Admin().execute(args);
}
@@ -210,6 +216,9 @@ public class Admin implements KeywordExecutable {
ChangeSecretCommand changeSecretCommand = new ChangeSecretCommand();
cl.addCommand("changeSecret", changeSecretCommand);
+DeleteZooInstanceCommand deleteZooInstanceOpts = new
DeleteZooInstanceCommand();
+cl.addCommand("deleteZooInstance", deleteZooInstanceOpts);
+
ListInstancesCommand listIntancesOpts = new ListInstancesCommand();
cl.addCommand("listInstances", listIntancesOpts);
@@ -291,6 +300,8 @@ public class Admin implements KeywordExecutable {
verifyTabletAssignmentsOpts.verbose);
} else if (cl.getParsedCommand().equals("changeSecret")) {
ChangeSecret.changeSecret(context, conf);
+ } else if (cl.getParsedCommand().equals("deleteZooInstance")) {
+DeleteZooInstance.deleteZooInstance(deleteZooInstanceOpts.instance);
} else {
everything = cl.getParsedCommand().equals("stopAll");
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
index 4c673ec1f1..d92a9245d4 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.server.util;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.util.HashSet;
+import java.util.Objects;
import java.util.Set;
import org.apache.accumulo.core.Constants;
@@ -52,35 +53,40 @@ public class DeleteZooInstance {
}
}
- /**
- * @param args
- * : the name or UUID of the instance to be deleted
- */
- public static void main(String[] args) throws Exception {
-Opts opts = new Opts();
-opts.parseArgs(DeleteZooInstance.class.getName(), args);
+ public static void deleteZooInstance(final String instance) throws Exception
{
+Objects.requireNonNull(instance, "Instance must not be null");
var zk = new ZooReaderWriter(SiteConfiguration.auto());
// try instance name:
Set instances = new HashSet<>(zk.getChildren(Constants.ZROOT +
Constants.ZINSTANCES));
Set uuids = new HashSet<>(zk.getChildren(Constants.ZROOT));
uuids.remove("instances");
-if (instances.contains(opts.instance)) {
- String path = Constants.ZROOT + Constants.ZINSTANCES + "/" +
opts.instance;
+if (instances.contains(instance)) {
+ String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
byte[] data = zk.getData(path);
deleteRetry(zk, path);
deleteRetry(zk, Constants.ZROOT + "/" + new String(data, UTF_8));
-} else if (uuids.contains(opts.instance)) {
+} else if (uuids.contains(instance)) {
// look for the real instance name
- for (String instance : instances) {
-String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + instance;
+ for (String zkInstance : instances) {
+String path = Constants.ZROOT + Constants.ZINSTANCES + "/" +
zkInstance;
[accumulo] branch main updated: Adding ChangeSecret utility to the accumulo admin command (#2814)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 93acb14430 Adding ChangeSecret utility to the accumulo admin command
(#2814)
93acb14430 is described below
commit 93acb14430096c9dcdf57248afa2c0bfbc1590f9
Author: Christopher L. Shannon
AuthorDate: Fri Aug 12 07:44:57 2022 -0400
Adding ChangeSecret utility to the accumulo admin command (#2814)
Issue #2807
---
.../src/main/java/org/apache/accumulo/server/util/Admin.java | 12
.../java/org/apache/accumulo/server/util/ChangeSecret.java | 11 ---
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index b7fe030f5c..053e0c197b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -169,6 +169,13 @@ public class Admin implements KeywordExecutable {
boolean verbose = false;
}
+ /**
+ * @since 2.1.0
+ */
+ @Parameters(
+ commandDescription = "Changes the unique secret given to the instance
that all servers must know.")
+ static class ChangeSecretCommand {}
+
public static void main(String[] args) {
new Admin().execute(args);
}
@@ -200,6 +207,9 @@ public class Admin implements KeywordExecutable {
CheckTabletsCommand checkTabletsCommand = new CheckTabletsCommand();
cl.addCommand("checkTablets", checkTabletsCommand);
+ChangeSecretCommand changeSecretCommand = new ChangeSecretCommand();
+cl.addCommand("changeSecret", changeSecretCommand);
+
ListInstancesCommand listIntancesOpts = new ListInstancesCommand();
cl.addCommand("listInstances", listIntancesOpts);
@@ -279,6 +289,8 @@ public class Admin implements KeywordExecutable {
} else if (cl.getParsedCommand().equals("verifyTabletAssigns")) {
VerifyTabletAssignments.verifyTableAssignments(opts.getClientProps(),
verifyTabletAssignmentsOpts.verbose);
+ } else if (cl.getParsedCommand().equals("changeSecret")) {
+ChangeSecret.changeSecret(context, conf);
} else {
everything = cl.getParsedCommand().equals("stopAll");
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index daa17c18fd..adfdc98321 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.trace.TraceUtil;
@@ -56,14 +57,18 @@ public class ChangeSecret {
public static void main(String[] args) throws Exception {
var siteConfig = SiteConfiguration.auto();
-var hadoopConf = new Configuration();
-
ServerUtilOpts opts = new ServerUtilOpts();
opts.parseArgs(ChangeSecret.class.getName(), args);
ServerContext context = opts.getServerContext();
+changeSecret(context, siteConfig);
+ }
+
+ public static void changeSecret(final ServerContext context, final
AccumuloConfiguration conf)
+ throws Exception {
+
try (var fs = context.getVolumeManager()) {
- ServerDirs serverDirs = new ServerDirs(siteConfig, hadoopConf);
+ ServerDirs serverDirs = new ServerDirs(conf, new Configuration());
verifyHdfsWritePermission(serverDirs, fs);
String oldPass = String.valueOf(System.console().readPassword("Old
secret: "));
[accumulo] branch main updated (06907260e5 -> f3952d0b1f)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 06907260e5 Modify exception handling in new ServerAmplImpl methods for ScanServers (#2866) add f3952d0b1f Fix args in ImportTableCommand. Closes #2869 (#2871) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/shell/commands/ImportTableCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] branch main updated: Add multiple dirs description to ImportTableCommand (#2862)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 428c5f5220 Add multiple dirs description to ImportTableCommand (#2862)
428c5f5220 is described below
commit 428c5f52203ee2e5bf5859ffdcc09d057cb50a01
Author: Mike Miller
AuthorDate: Wed Aug 10 14:48:32 2022 +
Add multiple dirs description to ImportTableCommand (#2862)
---
.../java/org/apache/accumulo/shell/commands/ImportTableCommand.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportTableCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportTableCommand.java
index 68b41c2fbd..6fe344b538 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportTableCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportTableCommand.java
@@ -44,7 +44,7 @@ public class ImportTableCommand extends Command {
@Override
public String usage() {
-return getName() + " ";
+return getName() + " { }";
}
@Override
[accumulo] branch main updated (160c36c7c6 -> 2c0cf177f6)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 160c36c7c6 Adding VerifyTabletAssignments utility to accumulo admin command (#2818) add 2c0cf177f6 Fix fate debug message. Closes #2838 (#2845) No new revisions were added by this update. Summary of changes: core/src/main/java/org/apache/accumulo/fate/Fate.java | 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-)
[accumulo] branch main updated (72626307c9 -> 160c36c7c6)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 72626307c9 Use `==` for enum equality add 160c36c7c6 Adding VerifyTabletAssignments utility to accumulo admin command (#2818) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/server/util/Admin.java | 14 +++ .../server/util/VerifyTabletAssignments.java | 27 ++ 2 files changed, 31 insertions(+), 10 deletions(-)
[accumulo] branch main updated (b049beefd6 -> 5b0ad9087f)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b049beefd6 Don't log full stack trace by default when unable to load compression codec (#2836) add 5b0ad9087f Fix names in FateLogger (#2829) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/core/logging/FateLogger.java| 12 ++-- core/src/main/java/org/apache/accumulo/fate/Fate.java| 8 core/src/main/java/org/apache/accumulo/fate/ZooStore.java| 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-)
[accumulo] branch main updated: Fix serializable warnings (#2832)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 54e847bdd4 Fix serializable warnings (#2832)
54e847bdd4 is described below
commit 54e847bdd42a0511234f8c0e997221b8a7277cfb
Author: Mike Miller
AuthorDate: Fri Jul 29 20:02:19 2022 +
Fix serializable warnings (#2832)
* Change private interface types to concrete classes to stop Java 18
warnings about non-serializable types
* Supports #2776
---
.../core/client/MutationsRejectedException.java | 21 +++--
.../accumulo/core/client/TimedOutException.java | 6 +++---
2 files changed, 14 insertions(+), 13 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
b/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
index 53378b3f0b..ffe39b9dc4 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/MutationsRejectedException.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.core.client;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,10 +38,10 @@ import org.apache.accumulo.core.data.TabletId;
public class MutationsRejectedException extends AccumuloException {
private static final long serialVersionUID = 1L;
- private List cvsl;
- private Map> af;
- private Collection es;
- private int unknownErrors;
+ private final ArrayList cvsl = new ArrayList<>();
+ private final HashMap> af = new HashMap<>();
+ private final HashSet es = new HashSet<>();
+ private final int unknownErrors;
/**
*
@@ -65,9 +66,9 @@ public class MutationsRejectedException extends
AccumuloException {
"# constraint violations : " + cvsList.size() + " security codes: " +
hashMap.toString()
+ " # server errors " + serverSideErrors.size() + " # exceptions
" + unknownErrors,
cause);
-this.cvsl = cvsList;
-this.af = hashMap;
-this.es = serverSideErrors;
+this.cvsl.addAll(cvsList);
+this.af.putAll(hashMap);
+this.es.addAll(serverSideErrors);
this.unknownErrors = unknownErrors;
}
@@ -93,9 +94,9 @@ public class MutationsRejectedException extends
AccumuloException {
super("# constraint violations : " + cvsList.size() + " security codes: "
+ format(hashMap, (ClientContext) client) + " # server errors " +
serverSideErrors.size()
+ " # exceptions " + unknownErrors, cause);
-this.cvsl = cvsList;
-this.af = hashMap;
-this.es = serverSideErrors;
+this.cvsl.addAll(cvsList);
+this.af.putAll(hashMap);
+this.es.addAll(serverSideErrors);
this.unknownErrors = unknownErrors;
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
b/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
index bd647099f9..4b40aa0c54 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/TimedOutException.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.core.client;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Set;
/**
@@ -27,7 +28,7 @@ import java.util.Set;
*/
public class TimedOutException extends RuntimeException {
- private Set timedoutServers;
+ private final HashSet timedoutServers = new HashSet<>();
private static final long serialVersionUID = 1L;
@@ -41,13 +42,12 @@ public class TimedOutException extends RuntimeException {
public TimedOutException(Set timedoutServers) {
super("Servers timed out " + shorten(timedoutServers));
-this.timedoutServers = timedoutServers;
+this.timedoutServers.addAll(timedoutServers);
}
public TimedOutException(String msg) {
super(msg);
-this.timedoutServers = Collections.emptySet();
}
public Set getTimedOutSevers() {
[accumulo] branch main updated (87649d46bf -> 388d3ffe11)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 87649d46bf Remove test for trace table (#2828) add 388d3ffe11 Create new methods in FastFormat for printing hex (#2826) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/core/util/FastFormat.java | 22 + .../java/org/apache/accumulo/fate/AdminUtil.java | 5 +-- .../main/java/org/apache/accumulo/fate/Fate.java | 7 ++--- .../java/org/apache/accumulo/fate/FateTxId.java| 4 ++- .../java/org/apache/accumulo/fate/ZooStore.java| 3 +- .../apache/accumulo/core/util/FastFormatTest.java | 36 ++ .../accumulo/manager/FateServiceHandler.java | 3 +- .../apache/accumulo/manager/tableOps/Utils.java| 16 +- .../manager/tableOps/compact/CompactRange.java | 5 +-- .../accumulo/shell/commands/FateCommand.java | 5 +-- .../accumulo/test/fate/zookeeper/FateIT.java | 13 11 files changed, 93 insertions(+), 26 deletions(-)
[accumulo] branch main updated: bump sec-bugs version (#2823)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 1748f101ef bump sec-bugs version (#2823) 1748f101ef is described below commit 1748f101efbc857453b48b9a3b2d686d86050bcd Author: EdColeman AuthorDate: Mon Jul 25 12:00:50 2022 + bump sec-bugs version (#2823) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ad199507e..032c0902d9 100644 --- a/pom.xml +++ b/pom.xml @@ -1591,7 +1591,7 @@ com.h3xstream.findsecbugs findsecbugs-plugin - 1.8.0 + 1.12.0
[accumulo] branch main updated: Make TabletFile constructor log at trace
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4b5bc76305 Make TabletFile constructor log at trace
4b5bc76305 is described below
commit 4b5bc76305ad1f5146299f8995337847cb8df8c0
Author: Mike Miller
AuthorDate: Thu Jul 21 15:05:41 2022 -0400
Make TabletFile constructor log at trace
---
core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
index c7e4a22926..de974ca687 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
@@ -56,7 +56,7 @@ public class TabletFile implements Comparable {
public TabletFile(Path metaPath) {
this.metaPath = Objects.requireNonNull(metaPath);
String errorMsg = "Missing or invalid part of tablet file metadata entry:
" + metaPath;
-log.debug("Parsing TabletFile from {}", metaPath);
+log.trace("Parsing TabletFile from {}", metaPath);
// use Path object to step backwards from the filename through all the
parts
this.fileName = metaPath.getName();
[accumulo] branch main updated: Updating DumpZookeeper utility to extend KeywordExecutable (#2807) (#2813)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ba049d5f10 Updating DumpZookeeper utility to extend KeywordExecutable
(#2807) (#2813)
ba049d5f10 is described below
commit ba049d5f10880d722453668bb3112b7d012db3e4
Author: Christopher L. Shannon
AuthorDate: Tue Jul 19 08:32:33 2022 -0400
Updating DumpZookeeper utility to extend KeywordExecutable (#2807) (#2813)
---
.../apache/accumulo/server/util/DumpZookeeper.java | 22 --
.../apache/accumulo/test/start/KeywordStartIT.java | 3 +++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
index 9a1907ab40..f3e92bd8e4 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/DumpZookeeper.java
@@ -25,15 +25,28 @@ import java.util.Base64;
import org.apache.accumulo.core.cli.ConfigOpts;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
-public class DumpZookeeper {
+@AutoService(KeywordExecutable.class)
+public class DumpZookeeper implements KeywordExecutable {
private static ZooReaderWriter zk = null;
+ @Override
+ public String keyword() {
+return "dump-zoo";
+ }
+
+ @Override
+ public String description() {
+return "Writes Zookeeper data as human readable or XML to a file.";
+ }
+
private static class Encoded {
public String encoding;
public String value;
@@ -53,7 +66,8 @@ public class DumpZookeeper {
boolean xml = false;
}
- public static void main(String[] args) throws KeeperException,
InterruptedException {
+ @Override
+ public void execute(String[] args) throws KeeperException,
InterruptedException {
Opts opts = new Opts();
opts.parseArgs(DumpZookeeper.class.getName(), args);
@@ -66,6 +80,10 @@ public class DumpZookeeper {
}
}
+ public static void main(String[] args) throws KeeperException,
InterruptedException {
+new DumpZookeeper().execute(args);
+ }
+
private static void writeXml(PrintStream out, String root)
throws KeeperException, InterruptedException {
write(out, 0, "", root);
diff --git
a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
index 60f9405032..c58b7a6008 100644
--- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -57,6 +57,7 @@ import
org.apache.accumulo.server.conf.util.ConfigPropertyUpgrader;
import org.apache.accumulo.server.init.Initialize;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.server.util.ConvertConfig;
+import org.apache.accumulo.server.util.DumpZookeeper;
import org.apache.accumulo.server.util.ECAdmin;
import org.apache.accumulo.server.util.Info;
import org.apache.accumulo.server.util.LoginProperties;
@@ -123,6 +124,7 @@ public class KeywordStartIT {
expectSet.put("config-upgrade", ConfigPropertyUpgrader.class);
expectSet.put("convert-config", ConvertConfig.class);
expectSet.put("create-token", CreateToken.class);
+expectSet.put("dump-zoo", DumpZookeeper.class);
expectSet.put("ec-admin", ECAdmin.class);
expectSet.put("gc", GCExecutable.class);
expectSet.put("generate-splits", GenerateSplits.class);
@@ -183,6 +185,7 @@ public class KeywordStartIT {
expectSet.add(Admin.class);
expectSet.add(CheckCompactionConfig.class);
expectSet.add(CreateToken.class);
+expectSet.add(DumpZookeeper.class);
expectSet.add(Info.class);
expectSet.add(Initialize.class);
expectSet.add(LoginProperties.class);
[accumulo] branch main updated: Mark ScanInterpreter related API as deprecated (#2787) (#2806)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 52f4c8de86 Mark ScanInterpreter related API as deprecated (#2787)
(#2806)
52f4c8de86 is described below
commit 52f4c8de862ef8658adc43f4b2afb8abff9cd2d0
Author: Christopher L. Shannon
AuthorDate: Tue Jul 19 08:27:45 2022 -0400
Mark ScanInterpreter related API as deprecated (#2787) (#2806)
---
.../src/main/java/org/apache/accumulo/core/conf/Property.java | 5 -
.../org/apache/accumulo/core/util/format/HexFormatter.java| 1 +
.../accumulo/core/util/interpret/DefaultScanInterpreter.java | 4
.../accumulo/core/util/interpret/HexScanInterpreter.java | 1 +
.../apache/accumulo/core/util/interpret/ScanInterpreter.java | 3 +++
.../apache/accumulo/core/util/format/HexFormatterTest.java| 1 +
shell/src/main/java/org/apache/accumulo/shell/Shell.java | 1 +
.../org/apache/accumulo/shell/commands/DeleteManyCommand.java | 1 +
.../org/apache/accumulo/shell/commands/FormatterCommand.java | 2 ++
.../java/org/apache/accumulo/shell/commands/GrepCommand.java | 1 +
.../apache/accumulo/shell/commands/InterpreterCommand.java| 4
.../org/apache/accumulo/shell/commands/MaxRowCommand.java | 1 +
.../java/org/apache/accumulo/shell/commands/ScanCommand.java | 11 ---
13 files changed, 32 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 82d0264284..100d8f4c69 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -1113,8 +1113,11 @@ public enum Property {
"1.3.5"),
TABLE_FORMATTER_CLASS("table.formatter", DefaultFormatter.class.getName(),
PropertyType.STRING,
"The Formatter class to apply on results in the shell", "1.4.0"),
+ @Deprecated(since = "2.1.0")
TABLE_INTERPRETER_CLASS("table.interepreter",
DefaultScanInterpreter.class.getName(),
- PropertyType.STRING, "The ScanInterpreter class to apply on scan
arguments in the shell",
+ PropertyType.STRING,
+ "The ScanInterpreter class to apply on scan arguments in the shell. "
+ + "Note that this property is deprecated and will be removed in a
future version.",
"1.5.0"),
TABLE_CLASSLOADER_CONTEXT("table.class.loader.context", "",
PropertyType.STRING,
"The context to use for loading per-table resources, such as iterators"
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java
b/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java
index e53a17805a..3d3fb3d011 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/format/HexFormatter.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text;
/**
* A simple formatter that print the row, column family, column qualifier, and
value as hex
*/
+@Deprecated(since = "2.1.0")
public class HexFormatter implements Formatter, ScanInterpreter {
private char[] chars =
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/interpret/DefaultScanInterpreter.java
b/core/src/main/java/org/apache/accumulo/core/util/interpret/DefaultScanInterpreter.java
index eba0a4653a..25086ff61a 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/interpret/DefaultScanInterpreter.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/interpret/DefaultScanInterpreter.java
@@ -20,6 +20,10 @@ package org.apache.accumulo.core.util.interpret;
import org.apache.hadoop.io.Text;
+/**
+ * @deprecated since 2.1.0 This will be removed in a future version in favor
of JShell
+ */
+@Deprecated(since = "2.1.0")
public class DefaultScanInterpreter implements ScanInterpreter {
@Override
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/interpret/HexScanInterpreter.java
b/core/src/main/java/org/apache/accumulo/core/util/interpret/HexScanInterpreter.java
index c57a2bb257..09a0400b1c 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/interpret/HexScanInterpreter.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/interpret/HexScanInterpreter.java
@@ -25,6 +25,7 @@ import org.apache.accumulo.core.util.format.HexFormatter;
* {@link HexFormatter} back to binary. The hex input can contain dashes
(because
* {@link HexFormatter} outputs dashes) which are ignored.
*/
+@Deprecated(since = "2.1.0")
public class HexScanInterpreter extends HexFormatter {
}
diff --git
a/core/src/main/java/org/apache/accumulo/
[accumulo] branch main updated: Create Reference interface for new GC classes (#2767)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new afe78566f8 Create Reference interface for new GC classes (#2767)
afe78566f8 is described below
commit afe78566f85f9e9fd5b43165305bc70cd3a9c878
Author: Mike Miller
AuthorDate: Thu Jun 23 21:59:43 2022 +
Create Reference interface for new GC classes (#2767)
* Create ReferenceFile class to implement Reference interface
* Make ReferenceDirectory extend ReferenceFile
* Create AllVolumesDiretory class to extend ReferenceFile and move
GcVolumeUtil method to class
* Comment and clean up GC code
* Update MetadataSchema.isValidDirCol regex to be more strict
* Make TableGroupWatcher use ReferenceFile for calls to GC
* Updates to various relevant tests
---
.../core/client/rfile/RFileWriterBuilder.java | 7 +--
.../org/apache/accumulo/core/gc/Reference.java | 61 +++---
.../accumulo/core/gc/ReferenceDirectory.java | 29 --
.../core/gc/{Reference.java => ReferenceFile.java} | 41 ++-
.../apache/accumulo/core/metadata/TabletFile.java | 8 ++-
.../accumulo/core/metadata/ValidationUtil.java | 24 +++--
.../accumulo/core/metadata/schema/Ample.java | 14 ++---
.../core/metadata/schema/MetadataSchema.java | 4 +-
.../accumulo/server/gc/AllVolumesDirectory.java| 53 +++
.../apache/accumulo/server/gc/GcVolumeUtil.java| 13 +
.../accumulo/server/metadata/ServerAmpleImpl.java | 12 ++---
.../accumulo/server/util/MetadataTableUtil.java| 9 ++--
.../main/java/org/apache/accumulo/gc/GCRun.java| 22 +---
.../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +++---
.../accumulo/gc/GarbageCollectionEnvironment.java | 7 +--
.../apache/accumulo/gc/GarbageCollectionTest.java | 4 +-
.../accumulo/gc/SimpleGarbageCollectorTest.java| 18 +++
.../accumulo/manager/TabletGroupWatcher.java | 19 ---
.../tableOps/bulkVer1/CleanUpBulkImport.java | 4 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 4 +-
.../accumulo/manager/upgrade/Upgrader9to10.java| 13 ++---
.../manager/upgrade/Upgrader9to10Test.java | 32 ++--
.../java/org/apache/accumulo/test/CloneIT.java | 52 +-
.../test/functional/GarbageCollectorIT.java| 28 +++---
.../test/upgrade/GCUpgrade9to10TestIT.java | 5 +-
25 files changed, 298 insertions(+), 201 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
index f26348d3b5..b2bd41165e 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.crypto.CryptoServiceFactory.ClassloaderType;
import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.metadata.ValidationUtil;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -131,11 +132,7 @@ class RFileWriterBuilder implements RFile.OutputArguments,
RFile.WriterFSOptions
@Override
public WriterFSOptions to(String filename) {
-Objects.requireNonNull(filename);
-if (!filename.endsWith(".rf")) {
- throw new IllegalArgumentException(
- "Provided filename (" + filename + ") does not end with '.rf'");
-}
+ValidationUtil.validateRFileName(filename);
this.out = new OutputArgs(filename);
return this;
}
diff --git a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
index f40424c62f..cdffbd7eeb 100644
--- a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
@@ -19,50 +19,29 @@
package org.apache.accumulo.core.gc;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
/**
- * A GC reference to a tablet file or directory.
+ * A GC reference used for collecting files and directories into a single
stream. The GC deals with
+ * two inputs conceptually: candidates and references. Candidates are files
that could be possibly
+ * be deleted if they are not defeated by a reference.
*/
-public class Reference implements Comparable {
- // parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003&q
[accumulo] branch main updated: Make ByteSequence Serializable (#2779)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 607003f3f1 Make ByteSequence Serializable (#2779)
607003f3f1 is described below
commit 607003f3f1c44c81bf98a83615349fb2130c4327
Author: Mike Miller
AuthorDate: Thu Jun 16 10:35:45 2022 +
Make ByteSequence Serializable (#2779)
---
core/src/main/java/org/apache/accumulo/core/data/ByteSequence.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/data/ByteSequence.java
b/core/src/main/java/org/apache/accumulo/core/data/ByteSequence.java
index f98d9ce89b..2d1e467a1a 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ByteSequence.java
@@ -18,12 +18,16 @@
*/
package org.apache.accumulo.core.data;
+import java.io.Serializable;
+
import org.apache.hadoop.io.WritableComparator;
/**
* A sequence of bytes.
*/
-public abstract class ByteSequence implements Comparable {
+public abstract class ByteSequence implements Comparable,
Serializable {
+
+ private static final long serialVersionUID = 1L;
/**
* Gets a byte within this sequence.
[accumulo] branch main updated: Make auths classes in Authorizations (#2777)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 418eee8fa0 Make auths classes in Authorizations (#2777) 418eee8fa0 is described below commit 418eee8fa0abed50f30cb6c97184964f924d2c67 Author: Mike Miller AuthorDate: Wed Jun 15 13:37:59 2022 + Make auths classes in Authorizations (#2777) * Change private member types from interfaces to classes to get rid of Java 18 warnings * For #2777 --- .../main/java/org/apache/accumulo/core/security/Authorizations.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java index 1df129b9a8..37e42eb1aa 100644 --- a/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java +++ b/core/src/main/java/org/apache/accumulo/core/security/Authorizations.java @@ -44,8 +44,8 @@ public class Authorizations implements Iterable, Serializable, Authoriza private static final long serialVersionUID = 1L; - private Set auths = new HashSet<>(); - private List authsList = new ArrayList<>(); // sorted order + private final HashSet auths = new HashSet<>(); + private final ArrayList authsList = new ArrayList<>(); // sorted order /** * An empty set of authorizations.
[accumulo] branch main updated: Log RootTabletMetadata object creation at trace (#2768)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d5f81877fc Log RootTabletMetadata object creation at trace (#2768)
d5f81877fc is described below
commit d5f81877fcc794c8158f38b840d02331e3c563dc
Author: Mike Miller
AuthorDate: Fri Jun 10 16:15:20 2022 +
Log RootTabletMetadata object creation at trace (#2768)
---
.../org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
index bbd85b6305..c1e420164d 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
@@ -97,7 +97,7 @@ public class RootTabletMetadata {
private final Data data;
public RootTabletMetadata(String json) {
-log.debug("Creating root tablet metadata from stored JSON: {}", json);
+log.trace("Creating root tablet metadata from stored JSON: {}", json);
this.data = gson.fromJson(json, Data.class);
checkArgument(data.version == VERSION, "Invalid Root Table Metadata JSON
version %s",
data.version);
[accumulo] branch main updated: Fix Reference in TabletGroupWatcher (#2764)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 0031befaa9 Fix Reference in TabletGroupWatcher (#2764)
0031befaa9 is described below
commit 0031befaa9253b31351d3d626fd96a51e1630b7b
Author: Mike Miller
AuthorDate: Fri Jun 10 11:24:17 2022 +
Fix Reference in TabletGroupWatcher (#2764)
* Closes #2763
* Move tabletDir variable down to ReferenceDirectory
---
core/src/main/java/org/apache/accumulo/core/gc/Reference.java | 2 --
.../src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java | 2 ++
.../src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java | 4 +---
.../java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java | 2 +-
4 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
index ec820b2a6a..cbc3a28193 100644
--- a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
@@ -27,7 +27,6 @@ import
org.apache.accumulo.core.metadata.schema.MetadataSchema;
public class Reference implements Comparable {
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
public final TableId tableId; // 2a
- public final String tabletDir; // t-0003
// the exact string that is stored in the metadata
public final String metadataEntry;
@@ -36,7 +35,6 @@ public class Reference implements Comparable {
MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(tableId.canonical());
this.tableId = tableId;
this.metadataEntry = metadataEntry;
-this.tabletDir = metadataEntry;
}
@Override
diff --git
a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
index 8a9ce21899..67d8619d8e 100644
--- a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
@@ -24,8 +24,10 @@ import org.apache.accumulo.core.data.TableId;
* Part of the Tablet File path that is definitely a directory.
*/
public class ReferenceDirectory extends Reference {
+ public final String tabletDir; // t-0003
public ReferenceDirectory(TableId tableId, String dirName) {
super(tableId, dirName);
+this.tabletDir = dirName;
}
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index cd42491444..4e4ef860a3 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -53,7 +53,6 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.gc.Reference;
-import org.apache.accumulo.core.gc.ReferenceDirectory;
import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.manager.thrift.ManagerState;
@@ -642,8 +641,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
Key key = entry.getKey();
if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
var stf = new
StoredTabletFile(key.getColumnQualifierData().toString());
- var tabletDirectory = new ReferenceDirectory(stf.getTableId(),
stf.getTabletDir());
- datafilesAndDirs.add(tabletDirectory);
+ datafilesAndDirs.add(new Reference(stf.getTableId(),
stf.getMetaUpdateDelete()));
if (datafilesAndDirs.size() > 1000) {
ample.putGcFileAndDirCandidates(extent.tableId(),
datafilesAndDirs);
datafilesAndDirs.clear();
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
index 60e196d11d..67f67340f9 100644
---
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
@@ -98,7 +98,7 @@ public class Upgrader9to10Test {
private void compareReferences(Reference ref1, Reference ref2) {
assertEquals(ref1.metadataEntry, ref2.metadataEntry);
-assertEquals(ref1.tabletDir, ref2.tabletDir);
+assertEquals(ref1.tableId, ref2.tableId);
}
@Test
[accumulo] branch main updated: Use Reference class in GC, Ample & TabletGroupWatcher (#2750)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 37e1c6f275 Use Reference class in GC, Ample & TabletGroupWatcher
(#2750)
37e1c6f275 is described below
commit 37e1c6f275cca0b118ed6ab631345c0699515552
Author: Mike Miller
AuthorDate: Wed Jun 8 18:38:23 2022 +
Use Reference class in GC, Ample & TabletGroupWatcher (#2750)
* Replace uses of String with Reference to prevent errors with
over use of String in GC, Ample and TabletGroupWatcher
* Make Reference comparable and implement required methods
* Rename RelativeTabletDirectory to ReferenceDirectory
* Rename TabletFileUtil to ValidationUtil and add new method for Reference
* Make TabletDirectory its own independent class and clean it up
* Create package org.apache.accumulo.core.gc and move Reference and
ReferenceDirectory to the new package
* Update GcVolumeUtil, MetadataTableUtil, CleanUpBulkImport to use
Reference class
* Update Upgrader and tests
Co-authored-by: Dave Marion
---
.../accumulo/core/{metadata => gc}/Reference.java | 35 ++--
.../ReferenceDirectory.java} | 6 ++--
.../accumulo/core/metadata/TabletDirectory.java| 20 ++--
.../apache/accumulo/core/metadata/TabletFile.java | 2 +-
.../{TabletFileUtil.java => ValidationUtil.java} | 10 --
.../accumulo/core/metadata/schema/Ample.java | 13
.../apache/accumulo/server/gc/GcVolumeUtil.java| 8 +++--
.../accumulo/server/metadata/ServerAmpleImpl.java | 15 +
.../apache/accumulo/server/util/LocalityCheck.java | 4 +--
.../accumulo/server/util/MetadataTableUtil.java| 12 +++
.../server/util/RemoveEntriesForMissingFiles.java | 4 +--
.../main/java/org/apache/accumulo/gc/GCRun.java| 12 +++
.../accumulo/gc/GarbageCollectionAlgorithm.java| 8 ++---
.../accumulo/gc/GarbageCollectionEnvironment.java | 2 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 6 ++--
.../accumulo/gc/SimpleGarbageCollectorTest.java| 16 -
.../accumulo/manager/TabletGroupWatcher.java | 30 +
.../tableOps/bulkVer1/CleanUpBulkImport.java | 3 +-
.../manager/tableOps/bulkVer1/CopyFailed.java | 4 +--
.../tableOps/bulkVer2/CleanUpBulkImport.java | 3 +-
.../tableOps/tableExport/WriteExportFiles.java | 4 +--
.../accumulo/manager/upgrade/Upgrader9to10.java| 28
.../manager/upgrade/Upgrader9to10Test.java | 38 +++---
.../test/functional/GarbageCollectorIT.java| 5 ++-
.../test/upgrade/GCUpgrade9to10TestIT.java | 3 +-
25 files changed, 191 insertions(+), 100 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
similarity index 64%
rename from core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
rename to core/src/main/java/org/apache/accumulo/core/gc/Reference.java
index 0254220040..ec820b2a6a 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.core.metadata;
+package org.apache.accumulo.core.gc;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
/**
- * A reference to a tablet file or directory.
+ * A GC reference to a tablet file or directory.
*/
-public class Reference {
+public class Reference implements Comparable {
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
public final TableId tableId; // 2a
public final String tabletDir; // t-0003
@@ -38,4 +38,33 @@ public class Reference {
this.metadataEntry = metadataEntry;
this.tabletDir = metadataEntry;
}
+
+ @Override
+ public int compareTo(Reference that) {
+if (equals(that)) {
+ return 0;
+} else {
+ return this.metadataEntry.compareTo(that.metadataEntry);
+}
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+if (this == obj)
+ return true;
+if (obj == null)
+ return false;
+if (getClass() != obj.getClass())
+ return false;
+Reference other = (Reference) obj;
+if (metadataEntry == null) {
+ return other.metadataEntry == null;
+} else
+ return metadataEntry.equals(other.metadataEntry);
+ }
+
+ @Override
+ public int hashCode() {
+return this.metadataEntry.hashCode();
+ }
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
b/core/src/main/java/org
[accumulo] branch main updated: Add Verify & fix test in NamespaceConfigurationTest (#2756)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 61b107d876 Add Verify & fix test in NamespaceConfigurationTest (#2756)
61b107d876 is described below
commit 61b107d876e2427b5fd7c1536036e1a050732ac5
Author: Mike Miller
AuthorDate: Tue Jun 7 10:58:26 2022 +
Add Verify & fix test in NamespaceConfigurationTest (#2756)
* Add Verify calls to NamespaceConfigurationTest
* Fix the testGet_InParent test in NamespaceConfigurationTest
Co-authored-by: Christopher Tubbs
---
.../server/conf/NamespaceConfigurationTest.java | 20
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
index 621e10f42f..47d2f769b0 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/conf/NamespaceConfigurationTest.java
@@ -37,6 +37,7 @@ import java.util.function.Predicate;
import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.NamespaceId;
@@ -91,6 +92,7 @@ public class NamespaceConfigurationTest {
NamespaceConfiguration nsConfig = new NamespaceConfiguration(context,
NSID, parent);
assertEquals(NSID, nsConfig.getNamespaceId());
assertEquals(parent, nsConfig.getParent());
+verify(propStore, context);
}
@Test
@@ -99,21 +101,21 @@ public class NamespaceConfigurationTest {
assertEquals("sekrit", nsConfig.get(Property.INSTANCE_SECRET));
-verify(propStore);
+verify(propStore, context);
}
@Test
public void testGet_InParent() {
-Property p = Property.INSTANCE_SECRET;
-
-expect(parent.get(p.getKey())).andReturn("sekrit");
-replay(parent);
+String randomKey = UUID.randomUUID().toString();
+String customTablePropKey = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+ randomKey;
+String expectedValue = "thisIsTheExpectedValue";
+ConfigurationCopy parent = new
ConfigurationCopy(Map.of(customTablePropKey, expectedValue));
nsConfig = new NamespaceConfiguration(context, NSID, parent);
-assertEquals("sekrit", nsConfig.get(Property.INSTANCE_SECRET));
+assertEquals(expectedValue, nsConfig.get(customTablePropKey));
-// TODO Need to check parent and accessor usage
+verify(propStore, context);
}
@Test
@@ -129,6 +131,7 @@ public class NamespaceConfigurationTest {
NamespaceConfiguration accumuloNsConfig =
new NamespaceConfiguration(context, Namespace.ACCUMULO.id(), parent);
assertNull(accumuloNsConfig.get(Property.INSTANCE_SECRET));
+verify(propStore, context);
}
@Test
@@ -160,6 +163,7 @@ public class NamespaceConfigurationTest {
assertEquals("tock", props.get("tick"));
assertEquals("bark", props.get("dog"));
assertEquals("meow", props.get("cat"));
+verify(propStore, parent, context);
}
@Test
@@ -170,6 +174,6 @@ public class NamespaceConfigurationTest {
assertEquals("sekrit", value);
nsConfig.invalidateCache();
-verify(propStore);
+verify(propStore, context);
}
}
[accumulo-proxy] branch main updated (60a8d9a -> 95f9c90)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-proxy.git from 60a8d9a Merge pull request #30 from apache/dependabot/maven/commons-io-commons-io-2.7 add caf6e13 Bump accumulo-core from 2.0.0 to 2.0.1 new 95f9c90 Merge pull request #29 from apache/dependabot/maven/org.apache.accumulo-accumulo-core-2.0.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo-proxy] 01/01: Merge pull request #29 from apache/dependabot/maven/org.apache.accumulo-accumulo-core-2.0.1
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-proxy.git commit 95f9c908bc086a049c322191ec9a8e767f886773 Merge: 60a8d9a caf6e13 Author: Mike Miller AuthorDate: Wed Jun 1 13:30:17 2022 + Merge pull request #29 from apache/dependabot/maven/org.apache.accumulo-accumulo-core-2.0.1 Bump accumulo-core from 2.0.0 to 2.0.1 pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo-proxy] branch main updated (a1905be -> 60a8d9a)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-proxy.git from a1905be Bump guava from 28.2-jre to 29.0-jre (#28) add d07a64a Bump commons-io from 2.6 to 2.7 new 60a8d9a Merge pull request #30 from apache/dependabot/maven/commons-io-commons-io-2.7 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo-proxy] 01/01: Merge pull request #30 from apache/dependabot/maven/commons-io-commons-io-2.7
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-proxy.git commit 60a8d9adf8023e2adaf380873ebc4878f8f0a3e5 Merge: a1905be d07a64a Author: Mike Miller AuthorDate: Wed Jun 1 13:30:00 2022 + Merge pull request #30 from apache/dependabot/maven/commons-io-commons-io-2.7 Bump commons-io from 2.6 to 2.7 pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] branch main updated: Log error in Admin for no servers (#2748)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 62aa21e40e Log error in Admin for no servers (#2748)
62aa21e40e is described below
commit 62aa21e40ebdcfd0a3c1d864d6adfe2c3de2de27
Author: Mike Miller
AuthorDate: Tue May 31 16:08:07 2022 +
Log error in Admin for no servers (#2748)
---
.../base/src/main/java/org/apache/accumulo/server/util/Admin.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 7ebefc472e..912c5113c8 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -115,7 +115,7 @@ public class Admin implements KeywordExecutable {
@Parameters(commandDescription = "stop the master (DEPRECATED -- use
stopManager instead)")
static class StopMasterCommand {}
- @Parameters(commandDescription = "stop all the servers")
+ @Parameters(commandDescription = "stop all of the tablets servers and
manager")
static class StopAllCommand {}
@Parameters(commandDescription = "list Accumulo instances in zookeeper")
@@ -379,6 +379,10 @@ public class Admin implements KeywordExecutable {
log.info("No managers running. Not attempting safe unload of tserver.");
return;
}
+if (servers.isEmpty()) {
+ log.error("No tablet servers provided.");
+ return;
+}
final String zTServerRoot = getTServersZkPath(context);
final ZooCache zc = context.getZooCache();
[accumulo] branch main updated (cf50b72bca -> fdd455ddd3)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from cf50b72bca Add check to Admin (#2743) add fdd455ddd3 Add new test to FateCommandTest (#2742) No new revisions were added by this update. Summary of changes: .../accumulo/shell/commands/FateCommand.java | 12 ++-- .../accumulo/shell/commands/FateCommandTest.java | 67 +++--- 2 files changed, 64 insertions(+), 15 deletions(-)
[accumulo] branch main updated: Add check to Admin (#2743)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new cf50b72bca Add check to Admin (#2743)
cf50b72bca is described below
commit cf50b72bca7729120dddc2e94396ef63137948fd
Author: Mike Miller
AuthorDate: Tue May 31 10:54:51 2022 +
Add check to Admin (#2743)
* Closes #1753
* Add check to prevent all tservers from being shutdown, unless forced
---
.../base/src/main/java/org/apache/accumulo/server/util/Admin.java | 8
1 file changed, 8 insertions(+)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index 43f79fe021..7ebefc472e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -379,9 +379,17 @@ public class Admin implements KeywordExecutable {
log.info("No managers running. Not attempting safe unload of tserver.");
return;
}
+
final String zTServerRoot = getTServersZkPath(context);
final ZooCache zc = context.getZooCache();
+List runningServers;
+
for (String server : servers) {
+ runningServers = context.instanceOperations().getTabletServers();
+ if (runningServers.size() == 1 && !force) {
+log.info("Only 1 tablet server running. Not attempting shutdown of
{}", server);
+return;
+ }
for (int port :
context.getConfiguration().getPort(Property.TSERV_CLIENTPORT)) {
HostAndPort address = AddressUtil.parseAddress(server, port);
final String finalServer =
[accumulo-website] branch main updated: Fix Accumulo tour page 1 markdown code (#318)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 5ca0cecc Fix Accumulo tour page 1 markdown code (#318)
5ca0cecc is described below
commit 5ca0cecc8241652edb884dba14b7817127cc7904
Author: Luke Foster <84727868+foste...@users.noreply.github.com>
AuthorDate: Fri May 27 09:02:05 2022 -0400
Fix Accumulo tour page 1 markdown code (#318)
---
tour/getting-started.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tour/getting-started.md b/tour/getting-started.md
index 64791679..b7213132 100644
--- a/tour/getting-started.md
+++ b/tour/getting-started.md
@@ -19,7 +19,7 @@ great here on the tour. Files and logs used by
[MiniAccumuloCluster] can be see
3. Modify the _exercise_ method to print a hello message. You will put your
code in this method for each lesson.
```java
- static void exercise(AccumuloClient client) {
+ static void exercise(AccumuloClient client) throws Exception {
// Write your code here
System.out.println("Hello world");
}
[accumulo] branch main updated: Refactor Root Tablet serialization code (#2718)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e12fcbefb6 Refactor Root Tablet serialization code (#2718)
e12fcbefb6 is described below
commit e12fcbefb6f2979c3aeabd5597f3799c107df1ba
Author: Mike Miller
AuthorDate: Fri May 27 11:26:59 2022 +
Refactor Root Tablet serialization code (#2718)
* Remove static state from RootTabletMetadata and RootGcCandidates
* Replaced static from() methods in RootTabletMetadata and RootGcCandidates
with constructors
* Added comment to AccumuloDataVersion javadoc
* Store entries natively in the simpler string-based multi-map, rather
than converting to key/value pairs and back again. Only convert to
key/value pairs when needed.
* Get rid of `new GsonBuilder().create()`. Even Gson's own javadoc
recommends `new Gson()` if all you want is the defaults.
Co-authored-by: Christopher Tubbs
---
.../core/clientImpl/bulk/BulkSerialize.java| 3 +-
.../schema/ExternalCompactionFinalState.java | 3 +-
.../schema/ExternalCompactionMetadata.java | 3 +-
.../core/metadata/schema/RootTabletMetadata.java | 224 +
.../core/metadata/schema/TabletMetadata.java | 2 +-
.../core/metadata/schema/TabletsMetadata.java | 9 +-
.../accumulo/server/AccumuloDataVersion.java | 2 +-
.../accumulo/server/init/ZooKeeperInitializer.java | 32 ++-
.../accumulo/server/metadata/RootGcCandidates.java | 106 +-
.../server/metadata/RootTabletMutatorImpl.java | 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 17 +-
.../manager/state/RootTabletStateStoreTest.java| 11 +-
.../accumulo/test/CountNameNodeOpsBulkIT.java | 3 +-
13 files changed, 198 insertions(+), 219 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
index 82fb70bc1b..f20efaa9ef 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
@@ -130,8 +130,7 @@ public class BulkSerialize {
final Path renamingFile = new Path(bulkDir, Constants.BULK_RENAME_FILE);
try (OutputStream fsOut = output.create(renamingFile);
BufferedWriter writer = new BufferedWriter(new
OutputStreamWriter(fsOut))) {
- Gson gson = new GsonBuilder().create();
- gson.toJson(oldToNewNameMap, writer);
+ new Gson().toJson(oldToNewNameMap, writer);
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
index 4a01d246a9..6e2aaa15ed 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
@@ -27,11 +27,10 @@ import org.apache.hadoop.io.Text;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
public class ExternalCompactionFinalState {
- private static final Gson GSON = new GsonBuilder().create();
+ private static final Gson GSON = new Gson();
public enum FinalState {
FINISHED, FAILED
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index b4a11c8084..fcd6bf2c3f 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -33,11 +33,10 @@ import
org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.hadoop.fs.Path;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
public class ExternalCompactionMetadata {
- private static final Gson GSON = new GsonBuilder().create();
+ private static final Gson GSON = new Gson();
private final Set jobFiles;
private final Set nextFiles;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
index c62b13145d..bbd85b6305 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
@@ -18,183 +18,153 @@
*/
package org.apache.accumulo.core.metadata.schema;
+import static
[accumulo] branch main updated: Protect against NPE in FateCommand.printTx (#2738)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3d17dd766d Protect against NPE in FateCommand.printTx (#2738)
3d17dd766d is described below
commit 3d17dd766d769a3a055bfc7d923e081589e92363
Author: Dave Marion
AuthorDate: Thu May 26 13:21:23 2022 -0400
Protect against NPE in FateCommand.printTx (#2738)
Closes #2736
---
.../main/java/org/apache/accumulo/shell/commands/FateCommand.java| 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
index fcdb70bb0a..b470474c42 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java
@@ -219,9 +219,8 @@ public class FateCommand extends Command {
ZooReaderWriter zk, ServiceLock.ServiceLockPath tableLocksPath, String[]
args, CommandLine cl,
boolean printStatus) throws InterruptedException, KeeperException,
IOException {
// Parse transaction ID filters for print display
-Set filterTxid = null;
-if (args.length >= 2) {
- filterTxid = new HashSet<>(args.length);
+Set filterTxid = new HashSet<>();
+if (args != null && args.length >= 2) {
for (int i = 1; i < args.length; i++) {
Long val = parseTxid(args[i]);
filterTxid.add(val);
[accumulo-website] branch main updated: Update release notes (#317)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 7746e11f Update release notes (#317)
7746e11f is described below
commit 7746e11f63c010bbf81ffe97a2bd422b7d0b39f0
Author: Mike Miller
AuthorDate: Wed May 25 18:20:40 2022 +
Update release notes (#317)
---
_posts/release/2022-04-27-accumulo-2.1.0.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/_posts/release/2022-04-27-accumulo-2.1.0.md
b/_posts/release/2022-04-27-accumulo-2.1.0.md
index 932f1f10..3ac3b44f 100644
--- a/_posts/release/2022-04-27-accumulo-2.1.0.md
+++ b/_posts/release/2022-04-27-accumulo-2.1.0.md
@@ -294,7 +294,10 @@ timing out. {% ghi 2599 %} {% ghi 2598 %}
### Other notable changes
+* {% ghi 1174 %} {% ghi 816 %} Abstract metadata and change root metadata
schema
* {% ghi 1309 %} Prevent cloning metadata table
+* {% ghi 1313 %} {% ghi 936 %} Store Root Tablet list of files in Zookeeper
+* {% ghi 1310 %} Examine WALs used by root tablet during GC
* {% ghi 1294 %} {% ghi 1299 %} Add optional -t tablename to importdirectory
shell command.
* {% ghi 1332 %} Disable FileSystemMonitor checks by default
* {% ghi 1345 %} {% ghi 1352 %} Allow gc initiated compaction as option.
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into main
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 6338319cf82cd93d16654679417ebb9ef1a33724 Merge: 811a9e33b4 c259914115 Author: Mike Miller AuthorDate: Tue May 24 13:56:59 2022 -0400 Merge remote-tracking branch 'upstream/1.10' into main
[accumulo] branch main updated (811a9e33b4 -> 6338319cf8)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 811a9e33b4 Simplify config handling for compaction tests (#2727) add c259914115 Disable Merging MINC by default (#2708) new 6338319cf8 Merge remote-tracking branch 'upstream/1.10' into main The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] branch 1.10 updated: Disable Merging MINC by default (#2708)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new c259914115 Disable Merging MINC by default (#2708)
c259914115 is described below
commit c259914115b80b4442eb9670608b6df1c6b2c0c0
Author: Mike Miller
AuthorDate: Tue May 24 17:38:39 2022 +
Disable Merging MINC by default (#2708)
* Merging minor compactions can lead to bad situations where a Tablet
can't flush. They have been removed in version 2.1. This change makes
the default value of table.compaction.minor.merge.file.size.max 1 byte,
which should disable merging minc by default.
* Here is the situation where a Tablet couldn't flush: An overwhelmed
tserver hosting a hot spot Tablet was hitting its max WAL limit
(TABLE_MINC_LOGS_MAX) so it was forcing a flush on the Tablet. The
TabletServerBatchWriter would try to flush its data by calling applyUpdates()
to the current commit session. The flush would timeout on the tserver,
presumably due to hitting max number of write threads and/or connection pools
filling up. A HoldTimeoutException would be thrown on the cli [...]
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 8dd0b6932c..6321a52322 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -683,10 +683,10 @@ public enum Property {
"After a tablet has been idle (no mutations) for this time period it may
have its "
+ "in-memory map flushed to disk in a minor compaction. There is no
guarantee an idle "
+ "tablet will be compacted."),
- TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max",
"0",
+ TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max",
"1",
PropertyType.MEMORY,
- "The max file size used for a merging minor compaction. The default
value"
- + " of 0 disables a max file size."),
+ "The max file size used for a merging minor compaction. The value of 0
is no max "
+ + "file size. The default value is 1 byte, which should disable
merging minor compactions."),
TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
"The maximum amount of memory that will be used to cache results of a
client query/scan. "
+ "Once this limit is reached, the buffered data is sent to the
client."),
[accumulo] branch main updated: Improve the madness in the GC (#2716)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 76aaba0ec4 Improve the madness in the GC (#2716)
76aaba0ec4 is described below
commit 76aaba0ec4e086ab535a1e88ba6ecb0f62481b2c
Author: Mike Miller
AuthorDate: Thu May 19 11:02:52 2022 +
Improve the madness in the GC (#2716)
* Add comments to makeRelative method and simplify
* Create new emptyPathsTest in GarbageCollectionTest
Co-authored-by: Christopher Tubbs
---
.../accumulo/gc/GarbageCollectionAlgorithm.java| 53 --
.../apache/accumulo/gc/GarbageCollectionTest.java | 22 +
2 files changed, 51 insertions(+), 24 deletions(-)
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 762c597675..b6e770001a 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -18,8 +18,10 @@
*/
package org.apache.accumulo.gc;
+import static java.util.Arrays.stream;
+import static java.util.function.Predicate.not;
+
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
@@ -52,59 +54,62 @@ public class GarbageCollectionAlgorithm {
private static final Logger log =
LoggerFactory.getLogger(GarbageCollectionAlgorithm.class);
+ /**
+ * This method takes a file or directory path and returns a relative path in
1 of 2 forms:
+ *
+ *
+ * 1- For files: table-id/tablet-directory/filename.rf
+ * 2- For directories: table-id/tablet-directory
+ *
+ *
+ * For example, for full file path like
hdfs://foo:6000/accumulo/tables/4/t0/F000.rf it will
+ * return 4/t0/F000.rf. For a directory that already is relative, like 4/t0,
it will just return
+ * the original path. This method will also remove prefixed relative paths
like ../4/t0/F000.rf
+ * and return 4/t0/F000.rf. It also strips out empty tokens from paths like
+ * hdfs://foo.com:6000/accumulo/tables/4//t0//F001.rf returning 4/t0/F001.rf.
+ */
private String makeRelative(String path, int expectedLen) {
String relPath = path;
+// remove prefixed old relative path
if (relPath.startsWith("../"))
relPath = relPath.substring(3);
+// remove trailing slash
while (relPath.endsWith("/"))
relPath = relPath.substring(0, relPath.length() - 1);
+// remove beginning slash
while (relPath.startsWith("/"))
relPath = relPath.substring(1);
-String[] tokens = relPath.split("/");
-
-// handle paths like a//b///c
-boolean containsEmpty = false;
-for (String token : tokens) {
- if (token.equals("")) {
-containsEmpty = true;
-break;
- }
-}
-
-if (containsEmpty) {
- ArrayList tmp = new ArrayList<>();
- for (String token : tokens) {
-if (!token.equals("")) {
- tmp.add(token);
-}
- }
-
- tokens = tmp.toArray(new String[tmp.size()]);
-}
+// Handle paths like a//b///c by dropping the empty tokens.
+String[] tokens =
stream(relPath.split("/")).filter(not(""::equals)).toArray(String[]::new);
if (tokens.length > 3 && path.contains(":")) {
+ // full file path like hdfs://foo:6000/accumulo/tables/4/t0/F000.rf
if (tokens[tokens.length - 4].equals(Constants.TABLE_DIR)
&& (expectedLen == 0 || expectedLen == 3)) {
+// return the last 3 tokens after tables, like 4/t0/F000.rf
relPath = tokens[tokens.length - 3] + "/" + tokens[tokens.length - 2]
+ "/"
+ tokens[tokens.length - 1];
} else if (tokens[tokens.length - 3].equals(Constants.TABLE_DIR)
&& (expectedLen == 0 || expectedLen == 2)) {
+// return the last 2 tokens after tables, like 4/t0
relPath = tokens[tokens.length - 2] + "/" + tokens[tokens.length - 1];
} else {
-throw new IllegalArgumentException(path);
+throw new IllegalArgumentException("Failed to make path relative. Bad
reference: " + path);
}
} else if (tokens.length == 3 && (expectedLen == 0 || expectedLen == 3)
&& !path.contains(":")) {
+ // we already have a relative path so return it, like 4/t0/F000.rf
relPath = tokens[0] + "/" + tokens[1] + "/" + tokens[2];
} else if (tokens.length == 2 && (expectedLen == 0 || expectedLen == 2)
&& !path.contains(":&
[accumulo] branch main updated: Create TabletDirectory classes (#2688)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6c5e5683eb Create TabletDirectory classes (#2688)
6c5e5683eb is described below
commit 6c5e5683eb8e267aba60e9e93af3ea642cfb9ddb
Author: Mike Miller
AuthorDate: Wed May 18 15:19:09 2022 +
Create TabletDirectory classes (#2688)
* Create TabletDirectory and RelativeTabletDirectory
* Move Reference class and improve it so other classes can be children
* Add validate method to StoredTabletFile
* Use TabletDirectory in TabletFile
* Refactor some GC code
---
.../apache/accumulo/core/metadata/Reference.java | 41 +
.../core/metadata/RelativeTabletDirectory.java | 31 +
.../accumulo/core/metadata/StoredTabletFile.java | 13 ++
.../accumulo/core/metadata/TabletDirectory.java| 51 ++
.../apache/accumulo/core/metadata/TabletFile.java | 26 +--
.../main/java/org/apache/accumulo/gc/GCRun.java| 8 ++--
.../accumulo/gc/GarbageCollectionAlgorithm.java| 29 ++--
.../accumulo/gc/GarbageCollectionEnvironment.java | 13 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 7 +--
9 files changed, 172 insertions(+), 47 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
b/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
new file mode 100644
index 00..0254220040
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/Reference.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+
+/**
+ * A reference to a tablet file or directory.
+ */
+public class Reference {
+ // parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
+ public final TableId tableId; // 2a
+ public final String tabletDir; // t-0003
+
+ // the exact string that is stored in the metadata
+ public final String metadataEntry;
+
+ public Reference(TableId tableId, String metadataEntry) {
+
MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(tableId.canonical());
+this.tableId = tableId;
+this.metadataEntry = metadataEntry;
+this.tabletDir = metadataEntry;
+ }
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
b/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
new file mode 100644
index 00..ae5d53cbc5
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/RelativeTabletDirectory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.data.TableId;
+
+/**
+ * Part of the Tablet File path that is definitely a directory.
+ */
+public class RelativeTabletDirectory extends Reference {
+
+ public RelativeTabletDirectory(TableId tableId, String dirName) {
+super(tableId, dirName);
+ }
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
b/core/src/main/java/org/a
[accumulo] branch 1.10 updated: General improvements to CI (#2715)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new c55821213f General improvements to CI (#2715)
c55821213f is described below
commit c55821213f202f5b131f4475c3d4a755942ac8d6
Author: Mike Miller
AuthorDate: Tue May 17 19:26:00 2022 +
General improvements to CI (#2715)
---
assemble/src/main/assemblies/component.xml | 3 ---
test/system/continuous/README.md| 21 +++--
...{batch_walkers.txt.example => batch_walkers.txt} | 0
...{continuous-env.sh.example => continuous-env.sh} | 17 ++---
.../{ingesters.txt.example => ingesters.txt}| 0
.../{scanners.txt.example => scanners.txt} | 0
.../continuous/{walkers.txt.example => walkers.txt} | 0
7 files changed, 25 insertions(+), 16 deletions(-)
diff --git a/assemble/src/main/assemblies/component.xml
b/assemble/src/main/assemblies/component.xml
index 3df09db612..b1522eb467 100644
--- a/assemble/src/main/assemblies/component.xml
+++ b/assemble/src/main/assemblies/component.xml
@@ -181,7 +181,6 @@
src/**
target/**
-**/continuous-env.sh
@@ -198,8 +197,6 @@
**/*.pl
**/*.pyc
**/*.pyo
-**/walkers.txt
-**/ingesters.txt
**/randomwalk/conf/logger.xml
**/randomwalk/conf/randomwalk.conf
**/randomwalk/conf/walkers
diff --git a/test/system/continuous/README.md b/test/system/continuous/README.md
index 31ee4bdafa..412ef62c62 100644
--- a/test/system/continuous/README.md
+++ b/test/system/continuous/README.md
@@ -21,14 +21,23 @@ Continuous Query and Ingest
This directory contains a suite of scripts for placing continuous query and
ingest load on accumulo. The purpose of these script is two-fold. First,
place continuous load on accumulo to see if breaks. Second, collect
-statistics in order to understand how accumulo behaves. To run these scripts
-copy all of the `.example` files and modify them. You can put these scripts in
-the current directory or define a `CONTINUOUS_CONF_DIR` where the files will be
+statistics in order to understand how accumulo behaves. To run these scripts:
+
+1) Modify the `*.txt` config files.
+2) Enter cluster values in `continuous-env.sh`
+
+You can put these scripts in the current directory or define a
`CONTINUOUS_CONF_DIR` where the files will be
read from. These scripts rely on `pssh`. Before running any script you may need
to use `pssh` to create the log directory on each machine (if you want it
local).
-Also, create the table "ci" before running. You can run
-`org.apache.accumulo.test.continuous.GenSplits` to generate splits points for a
-continuous ingest table.
+
+First, create the table "ci" before running:
+
+> accumulo shell -e "createtable ci" -u root
+
+Optional: generate split points for CI table:
+
+> $ accumulo org.apache.accumulo.test.continuous.GenSplits 20 > /tmp/splits
+> $ accumulo shell -e "addsplits -t ci -sf /tmp/splits" -u root
The following ingest scripts insert data into accumulo that will form a random
graph.
diff --git a/test/system/continuous/batch_walkers.txt.example
b/test/system/continuous/batch_walkers.txt
similarity index 100%
rename from test/system/continuous/batch_walkers.txt.example
rename to test/system/continuous/batch_walkers.txt
diff --git a/test/system/continuous/continuous-env.sh.example
b/test/system/continuous/continuous-env.sh
similarity index 97%
rename from test/system/continuous/continuous-env.sh.example
rename to test/system/continuous/continuous-env.sh
index 10cea328bc..4527a59f95 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh
@@ -13,7 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# set this to an absolute path that exist on every machine
+# modify these values to match cluster setup
+INSTANCE_NAME=instance
+ZOO_KEEPERS=localhost:2181
+USER=user
+PASS=pass
+
# Inherit values from environment if they are already set.
HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop}
HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_HOME}
@@ -22,14 +27,9 @@ ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}
JAVA_HOME=${JAVA_HOME:-/opt/java}
ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/opt/zookeeper}
-CONTINUOUS_LOG_DIR=$ACCUMULO_HOME/test/system/continuous/logs
-INSTANCE_NAME=instance
-ZOO_KEEPERS=zhost1,zhost2
+# users
ACCUMULO_USER=$(whoami)
HDFS_USER=$(whoami)
-USER=user
-PASS=pass
-TABLE=ci
#set debug to on to enable logging of accumulo client debugging
DEBUG_INGEST=off
@@ -137,3 +137,6 @@ BULK_MAP_TASKS=10
# number of key value pairs to generate per mapper
BULK_MAP_NODES=1
+# other sta
[accumulo] branch main updated: Rename some confusing GC methods (#2686)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7af40badbe Rename some confusing GC methods (#2686)
7af40badbe is described below
commit 7af40badbe25718fa5050bf883582e23d7b76631
Author: Mike Miller
AuthorDate: Tue May 10 10:46:57 2022 -0400
Rename some confusing GC methods (#2686)
---
server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java | 3 ++-
.../org/apache/accumulo/gc/GarbageCollectionAlgorithm.java | 14 +-
.../apache/accumulo/gc/GarbageCollectionEnvironment.java | 5 +++--
.../java/org/apache/accumulo/gc/GarbageCollectionTest.java | 2 +-
4 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
index a4a80d99d4..b864bb6cca 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@ -161,7 +161,8 @@ public class GCRun implements GarbageCollectionEnvironment {
}
@Override
- public void delete(SortedMap confirmedDeletes) throws
TableNotFoundException {
+ public void deleteConfirmedCandidates(SortedMap
confirmedDeletes)
+ throws TableNotFoundException {
final VolumeManager fs = context.getVolumeManager();
var metadataLocation = level == Ample.DataLevel.ROOT
? context.getZooKeeperRoot() + " for " + RootTable.NAME :
level.metaTable();
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 5dd6705d6e..2f4b5c85cc 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -110,7 +110,6 @@ public class GarbageCollectionAlgorithm {
}
private SortedMap makeRelative(Collection candidates)
{
-
SortedMap ret = new TreeMap<>();
for (String candidate : candidates) {
@@ -127,10 +126,7 @@ public class GarbageCollectionAlgorithm {
return ret;
}
- /**
- * Return the number of BLIP flags seen.
- */
- private void confirmDeletes(GarbageCollectionEnvironment gce,
+ private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
SortedMap candidateMap) {
Iterator iter = gce.getReferences().iterator();
while (iter.hasNext()) {
@@ -281,7 +277,7 @@ public class GarbageCollectionAlgorithm {
Span confirmDeletesSpan = TraceUtil.startSpan(this.getClass(),
"confirmDeletes");
try (Scope scope = confirmDeletesSpan.makeCurrent()) {
blips = removeBlipCandidates(gce, candidateMap);
- confirmDeletes(gce, candidateMap);
+ removeCandidatesInUse(gce, candidateMap);
confirmDeletesFromReplication(gce, candidateMap);
} catch (Exception e) {
TraceUtil.setException(confirmDeletesSpan, e, true);
@@ -292,11 +288,11 @@ public class GarbageCollectionAlgorithm {
return blips;
}
- private void deleteConfirmed(GarbageCollectionEnvironment gce,
+ private void deleteConfirmedCandidates(GarbageCollectionEnvironment gce,
SortedMap candidateMap) throws IOException,
TableNotFoundException {
Span deleteSpan = TraceUtil.startSpan(this.getClass(), "deleteFiles");
try (Scope deleteScope = deleteSpan.makeCurrent()) {
- gce.delete(candidateMap);
+ gce.deleteConfirmedCandidates(candidateMap);
} catch (Exception e) {
TraceUtil.setException(deleteSpan, e, true);
throw e;
@@ -342,7 +338,7 @@ public class GarbageCollectionAlgorithm {
long blips = confirmDeletesTrace(gce, candidateMap);
gce.incrementInUseStat(origSize - candidateMap.size());
-deleteConfirmed(gce, candidateMap);
+deleteConfirmedCandidates(gce, candidateMap);
return blips;
}
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
index add7c7f819..38b6ac5456 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java
@@ -95,12 +95,13 @@ public interface GarbageCollectionEnvironment {
/**
* Delete the given files from the provided {@link Map} of relative path to
absolute path for each
- * file that should be deleted
+ * file that should be deleted. The candidates should already be confirmed
for deletion.
*
* @param candidateMap
* A Map from relative path to absolute path for files to be
deleted.
*/
- void delete(SortedMap candidateMap) throws
TableNotFoundException;
+ void
[accumulo-testing] branch main updated: Drop Replication concurrent RW test and IT (#205)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 8d87c67 Drop Replication concurrent RW test and IT (#205)
8d87c67 is described below
commit 8d87c676c558ffe8fe38062a95c49d2e501942c3
Author: Mike Miller
AuthorDate: Mon May 9 14:00:03 2022 -0400
Drop Replication concurrent RW test and IT (#205)
* Closes #171
---
.../testing/randomwalk/concurrent/Replication.java | 204 -
.../randomwalk/ReplicationRandomWalkIT.java| 58 --
2 files changed, 262 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java
deleted file mode 100644
index 444d000..000
---
a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.testing.randomwalk.concurrent;
-
-import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-import static
org.apache.accumulo.core.conf.Property.MANAGER_REPLICATION_SCAN_INTERVAL;
-import static org.apache.accumulo.core.conf.Property.REPLICATION_NAME;
-import static org.apache.accumulo.core.conf.Property.REPLICATION_PEERS;
-import static org.apache.accumulo.core.conf.Property.REPLICATION_PEER_PASSWORD;
-import static org.apache.accumulo.core.conf.Property.REPLICATION_PEER_USER;
-import static
org.apache.accumulo.core.conf.Property.REPLICATION_WORK_ASSIGNMENT_SLEEP;
-import static
org.apache.accumulo.core.conf.Property.REPLICATION_WORK_PROCESSOR_DELAY;
-import static
org.apache.accumulo.core.conf.Property.REPLICATION_WORK_PROCESSOR_PERIOD;
-import static org.apache.accumulo.core.conf.Property.TABLE_REPLICATION;
-import static org.apache.accumulo.core.conf.Property.TABLE_REPLICATION_TARGET;
-
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.admin.InstanceOperations;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.conf.ClientProperty;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.testing.TestProps;
-import org.apache.accumulo.testing.randomwalk.RandWalkEnv;
-import org.apache.accumulo.testing.randomwalk.State;
-import org.apache.accumulo.testing.randomwalk.Test;
-import org.apache.hadoop.io.Text;
-
-@Deprecated
-public class Replication extends Test {
-
- final int ROWS = 1000;
- final int COLS = 50;
-
- @Override
- public void visit(State state, RandWalkEnv env, Properties props) throws
Exception {
-final AccumuloClient c = env.getAccumuloClient();
-final String instName =
ClientProperty.INSTANCE_NAME.getValue(c.properties());
-final String zookeepers =
ClientProperty.INSTANCE_ZOOKEEPERS.getValue(c.properties());
-final InstanceOperations iOps = c.instanceOperations();
-final TableOperations tOps = c.tableOperations();
-
-// Replicate to ourselves
-iOps.setProperty(REPLICATION_NAME.getKey(), instName);
-iOps.setProperty(REPLICATION_PEERS.getKey() + instName,
-"org.apache.accumulo.tserver.replication.AccumuloReplicaSystem," +
instName + ","
-+ zookeepers);
-iOps.setProperty(REPLICATION_PEER_USER.getKey() + instName,
env.getAccumuloUserName());
-iOps.setProperty(REPLICATION_PEER_PASSWORD.getKey() + instName,
env.getAccumuloPassword());
-// Tweak some replication parameters to make the rep
[accumulo] branch main updated (bbd32b1ad4 -> c356c93f5b)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from bbd32b1ad4 Bug fix for caption in tservers table in monitor (#2674) add c356c93f5b GC Blip Count Follow up Fixes (#2678) No new revisions were added by this update. Summary of changes: .../apache/accumulo/gc/GarbageCollectionAlgorithm.java | 12 .../org/apache/accumulo/gc/GarbageCollectionTest.java| 16 ++-- 2 files changed, 18 insertions(+), 10 deletions(-)
[accumulo] branch main updated: Clean up bulkImport js (#2675)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new c5ca1713d1 Clean up bulkImport js (#2675)
c5ca1713d1 is described below
commit c5ca1713d1f70b71effbfb1451a881590590437a
Author: Mike Miller
AuthorDate: Fri May 6 06:33:34 2022 -0400
Clean up bulkImport js (#2675)
* Refactor bulkImport.js to be formatted properly and use concise jquery
Co-authored-by: Dom G.
---
.../accumulo/monitor/resources/js/bulkImport.js| 139 ++---
.../accumulo/monitor/resources/js/functions.js | 8 +-
.../apache/accumulo/monitor/resources/js/global.js | 3 +
.../apache/accumulo/monitor/resources/js/server.js | 3 +-
.../accumulo/monitor/templates/bulkImport.ftl | 4 +-
5 files changed, 75 insertions(+), 82 deletions(-)
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
index bac1c7d581..5ef20d568c 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js
@@ -16,99 +16,88 @@
* specific language governing permissions and limitations
* under the License.
*/
+/* JSLint global definitions */
+/*global
+$, document, sessionStorage, clearTableBody, EMPTY_ROW_THREE_CELLS,
EMPTY_CELL, getBulkImports
+*/
"use strict";
-/**
- * Creates bulk import initial table
- */
-$(document).ready(function() {
- refreshBulkImport();
-});
-
-/**
- * Makes the REST calls, generates the tables with the new information
- */
-function refreshBulkImport() {
- getBulkImports().then(function() {
-refreshBulkImportTable();
-refreshServerBulkTable();
- });
-}
-
-/**
- * Used to redraw the page
- */
-function refresh() {
- refreshBulkImport();
-}
-
/**
* Generates the manager bulk import status table
*/
function refreshBulkImportTable() {
+$("#bulkListTable tbody").html(EMPTY_ROW_THREE_CELLS);
- clearTableBody('managerBulkImportStatus');
-
- /*
- * Get the bulk import value obtained earlier, if it doesn't exists,
- * create an empty array
- */
- var data = sessionStorage.bulkImports === undefined ?
- [] : JSON.parse(sessionStorage.bulkImports);
+// Get the bulk import data from the session
+var data = sessionStorage.bulkImports === undefined ?
+[] : JSON.parse(sessionStorage.bulkImports);
- /* If the data is empty, create an empty row, otherwise,
- * create the rows for the table
- */
- if (data.length === 0 || data.bulkImport.length === 0) {
-$('', {
- html: createEmptyRow(3, 'Empty')
-}).appendTo('#managerBulkImportStatus tbody');
- } else {
-$.each(data.bulkImport, function(key, val) {
- var items = [];
- items.push(createFirstCell(val.filename, val.filename));
- items.push(createRightCell(val.age, new Date(val.age)));
- items.push(createRightCell(val.state, val.state));
- $('', {
-html: items.join('')
- }).appendTo('#managerBulkImportStatus tbody');
+// If the data is empty, clear table, otherwise populate
+if (data.length === 0 || data.bulkImport.length === 0) {
+return;
+}
+console.log("Populate bulkListTable with " + sessionStorage.bulkImports);
+var tableBodyHtml = "";
+$.each(data.bulkImport, function (key, val) {
+console.log("Append row " + key + " " + JSON.stringify(val) + " to
bulkListTable");
+tableBodyHtml += "" + val.filename + "";
+tableBodyHtml += "" + new Date(val.age) + "";
+tableBodyHtml += "" + val.state + "";
});
- }
+$("#bulkListTable tbody").html(tableBodyHtml);
}
/**
- * Generates the bulk import status table
+ * Generates the bulkPerServerTable table
*/
function refreshServerBulkTable() {
+$("#bulkPerServerTable tbody").html(EMPTY_ROW_THREE_CELLS);
- clearTableBody('bulkImportStatus');
+// get the bulkImport data from sessionStorage
+var data = sessionStorage.bulkImports === undefined ?
+[] : JSON.parse(sessionStorage.bulkImports);
- /* Get the bulk import value obtained earlier, if it doesn't exists,
- * create an empty array
- */
- var data = sessionStorage.bulkImports === undefined ?
- [] : JSON.parse(sessionStorage.bulkImports);
+// if data is empty, log an error because that means no tablet servers
were found
+if (data.length === 0 || data.tabletServerBulkImport.length === 0) {
+console.error("No tablet servers.");
+return;
+}
+var tableBodyHtml
[accumulo] branch main updated: Add new blip count to GC (#2672)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 74b48da924 Add new blip count to GC (#2672)
74b48da924 is described below
commit 74b48da924ee5c2cdd378786f24275dd77266fc5
Author: Mike Miller
AuthorDate: Wed May 4 16:12:13 2022 -0400
Add new blip count to GC (#2672)
* Update GarbageCollectionTest to count blips
* Split up confirmDeletes into 3 methods
---
.../accumulo/core/gc/thrift/GcCycleStats.java | 104 ++-
core/src/main/thrift/gc.thrift | 1 +
.../accumulo/gc/GarbageCollectionAlgorithm.java| 114 -
.../apache/accumulo/gc/SimpleGarbageCollector.java | 7 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 7 +-
5 files changed, 175 insertions(+), 58 deletions(-)
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/gc/thrift/GcCycleStats.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/gc/thrift/GcCycleStats.java
index c28b60c15a..6aa14a0bfc 100644
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/gc/thrift/GcCycleStats.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/gc/thrift/GcCycleStats.java
@@ -34,6 +34,7 @@ public class GcCycleStats implements
org.apache.thrift.TBase byName = new
java.util.HashMap();
@@ -80,6 +83,8 @@ public class GcCycleStats implements
org.apache.thrift.TBase metaDataMap;
static {
@@ -143,6 +149,8 @@ public class GcCycleStats implements
org.apache.thrift.TBase candidateMap) throws TableNotFoundException {
-boolean checkForBulkProcessingFiles = false;
-Iterator relativePaths = candidateMap.keySet().iterator();
-while (!checkForBulkProcessingFiles && relativePaths.hasNext())
- checkForBulkProcessingFiles |=
-
relativePaths.next().toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
-
-if (checkForBulkProcessingFiles) {
- try (Stream blipStream = gce.getBlipPaths()) {
-Iterator blipiter = blipStream.iterator();
-
-// WARNING: This block is IMPORTANT
-// You MUST REMOVE candidates that are in the same folder as a bulk
-// processing flag!
-
-while (blipiter.hasNext()) {
- String blipPath = blipiter.next();
- blipPath = makeRelative(blipPath, 2);
-
- Iterator tailIter =
candidateMap.tailMap(blipPath).keySet().iterator();
-
- int count = 0;
-
- while (tailIter.hasNext()) {
-if (tailIter.next().startsWith(blipPath)) {
- count++;
- tailIter.remove();
-} else {
- break;
-}
- }
-
- if (count > 0)
-log.debug("Folder has bulk processing flag: {}", blipPath);
-}
- }
-}
-
+ SortedMap candidateMap) {
Iterator iter = gce.getReferences().iterator();
while (iter.hasNext()) {
Reference ref = iter.next();
@@ -200,17 +166,61 @@ public class GarbageCollectionAlgorithm {
String dir = reference.substring(0, reference.lastIndexOf('/'));
if (candidateMap.remove(dir) != null)
log.debug("Candidate was still in use: {}", reference);
+ }
+}
+ }
+
+ private long removeBlipCandidates(GarbageCollectionEnvironment gce,
+ SortedMap candidateMap) throws TableNotFoundException {
+boolean checkForBulkProcessingFiles = false;
+long blipCount = 0;
+Iterator relativePaths = candidateMap.keySet().iterator();
+
+while (!checkForBulkProcessingFiles && relativePaths.hasNext())
+ checkForBulkProcessingFiles |=
+
relativePaths.next().toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
+
+if (checkForBulkProcessingFiles) {
+ try (Stream blipStream = gce.getBlipPaths()) {
+Iterator blipiter = blipStream.iterator();
+
+// WARNING: This block is IMPORTANT
+// You MUST REMOVE candidates that are in the same folder as a bulk
+// processing flag!
+
+while (blipiter.hasNext()) {
+ String blipPath = blipiter.next();
+ blipPath = makeRelative(blipPath, 2);
+
+ Iterator tailIter =
candidateMap.tailMap(blipPath).keySet().iterator();
+
+ int count = 0;
+
+ while (tailIter.hasNext()) {
+if (tailIter.next().startsWith(blipPath)) {
+ count++;
+ tailIter.remove();
+} else {
+ break;
+}
+ }
+ if (count > 0) {
+log.debug("Folder has bulk processing flag: {}", blipPath);
+blipCount++;
+ }
+}
}
}
-confirmDeletesFromReplication(gce.getReplicationNeededIterator(),
-candidateMap.entrySet().iterator());
+return blipCount;
[accumulo] branch main updated: Create new internal GC class (#2670)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 16b5e3bb7b Create new internal GC class (#2670)
16b5e3bb7b is described below
commit 16b5e3bb7b3598896004f5648a9418f3e238d570
Author: Mike Miller
AuthorDate: Tue May 3 08:40:33 2022 -0400
Create new internal GC class (#2670)
* Pull out the GCEnv inner class of SimpleGarbageCollector into its own
class and call it GCRun.
* Keep the logic of GCEnv but make the stats private numbers to allow
dropping of synchronized block
* Create separate logger classes for each instance of GCRun
* Make GCRun return the stats gathered during that run
* Make SimpleGarbageCollector increment the current stats based on what
is returned in the GCRun class
---
.../main/java/org/apache/accumulo/gc/GCRun.java| 449 +
.../apache/accumulo/gc/SimpleGarbageCollector.java | 382 ++
.../accumulo/gc/SimpleGarbageCollectorTest.java| 6 +-
3 files changed, 484 insertions(+), 353 deletions(-)
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
new file mode 100644
index 00..a4a80d99d4
--- /dev/null
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.gc;
+
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SCANS;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.manager.state.tables.TableState;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.TabletFileUtil;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.threads.ThreadPools;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeUtil;
+import org.apache.accumulo.server.gc.GcVolumeUtil;
+import org.apache.accumulo.server.replication.proto.Replication;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * A single garbage collection performed on a table (Root, MD) or all User
tables.
+ */
+public class GCRun implements GarbageCollectionEnvironment {
+ private final Logger log;
+ private final Ample.DataLevel level;
+ private final S
[accumulo] branch main updated: Add js strict directive to each js file (#2664)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new bbcf67424d Add js strict directive to each js file (#2664) bbcf67424d is described below commit bbcf67424d320d72c41de71894d3954b5d1616a6 Author: Mike Miller AuthorDate: Fri Apr 29 16:21:12 2022 -0400 Add js strict directive to each js file (#2664) --- .../resources/org/apache/accumulo/monitor/resources/js/bulkImport.js | 1 + .../resources/org/apache/accumulo/monitor/resources/js/compactions.js| 1 + .../src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js| 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/functions.js | 1 + .../src/main/resources/org/apache/accumulo/monitor/resources/js/gc.js| 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/global.js| 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/listType.js | 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/manager.js | 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/navbar.js| 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/overview.js | 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/problems.js | 1 + .../resources/org/apache/accumulo/monitor/resources/js/replication.js| 1 + .../src/main/resources/org/apache/accumulo/monitor/resources/js/scans.js | 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/server.js| 1 + .../src/main/resources/org/apache/accumulo/monitor/resources/js/show.js | 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/summary.js | 1 + .../src/main/resources/org/apache/accumulo/monitor/resources/js/table.js | 1 + .../main/resources/org/apache/accumulo/monitor/resources/js/tservers.js | 1 + 18 files changed, 18 insertions(+) diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js index 53e58b..bac1c7d581 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/bulkImport.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +"use strict"; /** * Creates bulk import initial table diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js index f872e73b38..09ad311304 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +"use strict"; var compactionsList; /** diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js index dac238dbdd..556e2883b2 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +"use strict"; var coordinatorTable; var compactorsTable; diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js index 9363db104e..0c2fb7a0aa 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +"use strict"; // Suffixes for quantity var QUANTITY_SUFFIX = ['', 'K', 'M', 'B', 'T', 'e15', 'e18', 'e21']; diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/gc.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/gc.js index 3937df1f3a..4f14879a22 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/gc.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/gc.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +"use strict"; var gcTable; /** diff --git a/server/monitor/src/main/resources/org/
[accumulo] branch main updated: Upgrade to Bootstrap v3.4.1 and include missing map files (#2662)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d51e2f1c44 Upgrade to Bootstrap v3.4.1 and include missing map files
(#2662)
d51e2f1c44 is described below
commit d51e2f1c4455012b66d38600a3579b5d233fc4e3
Author: Mike Miller
AuthorDate: Fri Apr 29 06:51:51 2022 -0400
Upgrade to Bootstrap v3.4.1 and include missing map files (#2662)
* Upgrade to Bootstrap v3.4.1
* Add missing map files to fix console errors
* Update License file
---
LICENSE| 2 +-
.../external/bootstrap/css/bootstrap-theme.css | 200 ++---
.../external/bootstrap/css/bootstrap-theme.css.map | 1 +
.../resources/external/bootstrap/css/bootstrap.css | 985 +++--
.../external/bootstrap/css/bootstrap.css.map | 1 +
.../resources/external/bootstrap/js/bootstrap.js | 397 +++--
6 files changed, 934 insertions(+), 652 deletions(-)
diff --git a/LICENSE b/LICENSE
index 1ba4f1ca4a..ef53c6b0d0 100644
--- a/LICENSE
+++ b/LICENSE
@@ -308,7 +308,7 @@ Files:
* Dual licensed under the MIT and GPL licenses.
* http://benalman.com/about/license/
-## Bootstrap v3.3.7 (http://getbootstrap.com)
+## Bootstrap v3.4.1 (http://getbootstrap.com)
Files:
*
server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/bootstrap/**/*
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/bootstrap/css/bootstrap-theme.css
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/bootstrap/css/bootstrap-theme.css
index 31d8882661..ea33f76a77 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/bootstrap/css/bootstrap-theme.css
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/bootstrap/css/bootstrap-theme.css
@@ -1,6 +1,6 @@
/*!
- * Bootstrap v3.3.7 (http://getbootstrap.com)
- * Copyright 2011-2016 Twitter, Inc.
+ * Bootstrap v3.4.1 (https://getbootstrap.com/)
+ * Copyright 2011-2019 Twitter, Inc.
* Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
*/
.btn-default,
@@ -9,9 +9,9 @@
.btn-info,
.btn-warning,
.btn-danger {
- text-shadow: 0 -1px 0 rgba(0, 0, 0, .2);
- -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, .15), 0 1px 1px
rgba(0, 0, 0, .075);
- box-shadow: inset 0 1px 0 rgba(255, 255, 255, .15), 0 1px 1px
rgba(0, 0, 0, .075);
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
+ -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 1px
rgba(0, 0, 0, 0.075);
+ box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 1px rgba(0, 0, 0,
0.075);
}
.btn-default:active,
.btn-primary:active,
@@ -25,8 +25,8 @@
.btn-info.active,
.btn-warning.active,
.btn-danger.active {
- -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);
- box-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);
+ -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
+ box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
}
.btn-default.disabled,
.btn-primary.disabled,
@@ -47,7 +47,7 @@ fieldset[disabled] .btn-info,
fieldset[disabled] .btn-warning,
fieldset[disabled] .btn-danger {
-webkit-box-shadow: none;
- box-shadow: none;
+ box-shadow: none;
}
.btn-default .badge,
.btn-primary .badge,
@@ -62,15 +62,15 @@ fieldset[disabled] .btn-danger {
background-image: none;
}
.btn-default {
- text-shadow: 0 1px 0 #fff;
background-image: -webkit-linear-gradient(top, #fff 0%, #e0e0e0 100%);
- background-image: -o-linear-gradient(top, #fff 0%, #e0e0e0 100%);
+ background-image: -o-linear-gradient(top, #fff 0%, #e0e0e0 100%);
background-image: -webkit-gradient(linear, left top, left bottom,
from(#fff), to(#e0e0e0));
- background-image: linear-gradient(to bottom, #fff 0%, #e0e0e0 100%);
+ background-image: linear-gradient(to bottom, #fff 0%, #e0e0e0 100%);
filter:
progid:DXImageTransform.Microsoft.gradient(startColorstr='#',
endColorstr='#ffe0e0e0', GradientType=0);
filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
background-repeat: repeat-x;
border-color: #dbdbdb;
+ text-shadow: 0 1px 0 #fff;
border-color: #ccc;
}
.btn-default:hover,
@@ -106,9 +106,9 @@ fieldset[disabled] .btn-default.active {
}
.btn-primary {
background-image: -webkit-linear-gradient(top, #337ab7 0%, #265a88 100%);
- background-image: -o-linear-gradient(top, #337ab7 0%, #265a88 100%);
+ background-image: -o-linear-gradient(top, #337ab7 0%, #265a88 100%);
background-image: -webkit-gradient(linear, left top, left bottom,
from(#337ab7), to(#265a88));
- background-image: linear-gradient(to bottom, #337ab7 0%, #265a88
100%);
+ background-image: linear-gradient(to bottom
[accumulo] branch main updated: Fix js in server.js
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 375e1cc6c2 Fix js in server.js
375e1cc6c2 is described below
commit 375e1cc6c233f583df6e724b5a3ed4d456ce5152
Author: Mike Miller
AuthorDate: Tue Apr 26 05:46:10 2022 -0400
Fix js in server.js
---
.../main/resources/org/apache/accumulo/monitor/resources/js/server.js| 1 -
1 file changed, 1 deletion(-)
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
index 091f6c066a..84784055da 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
@@ -42,7 +42,6 @@ function refresh() {
* Populates the server details table
*/
function refreshDetailTable() {
-clearTableBody('tServerDetail');
var data = sessionStorage.server === undefined ?
[] : JSON.parse(sessionStorage.server);
if (data.length === 0 || data.details === undefined) {
[accumulo] branch main updated: Replace javascript code with jquery for server (#2650)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5cbe96a320 Replace javascript code with jquery for server (#2650)
5cbe96a320 is described below
commit 5cbe96a320dfe849ab50e18e77688138b0acc286
Author: Mike Miller
AuthorDate: Tue Apr 26 04:24:10 2022 -0400
Replace javascript code with jquery for server (#2650)
---
.../apache/accumulo/monitor/resources/js/server.js | 46 +++---
.../apache/accumulo/monitor/templates/server.ftl | 15 ++-
2 files changed, 28 insertions(+), 33 deletions(-)
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
index e09ae54321..091f6c066a 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/server.js
@@ -39,39 +39,23 @@ function refresh() {
}
/**
- * Generates the server details table
+ * Populates the server details table
*/
function refreshDetailTable() {
-
- clearTableBody('tServerDetail');
-
- var data = sessionStorage.server === undefined ?
- [] : JSON.parse(sessionStorage.server);
-
- var items = [];
-
- if (data.length === 0 || data.details === undefined) {
-items.push(createEmptyRow(5, 'Empty'));
- } else {
-items.push(createFirstCell(data.details.hostedTablets,
-bigNumberForQuantity(data.details.hostedTablets)));
-
-items.push(createRightCell(data.details.entries,
-bigNumberForQuantity(data.details.entries)));
-
-items.push(createRightCell(data.details.minors,
-bigNumberForQuantity(data.details.minors)));
-
-items.push(createRightCell(data.details.majors,
-bigNumberForQuantity(data.details.majors)));
-
-items.push(createRightCell(data.details.splits,
-bigNumberForQuantity(data.details.splits)));
- }
-
- $('', {
-html: items.join('')
- }).appendTo('#tServerDetail tbody');
+clearTableBody('tServerDetail');
+var data = sessionStorage.server === undefined ?
+[] : JSON.parse(sessionStorage.server);
+if (data.length === 0 || data.details === undefined) {
+$("#tServerDetail > tbody > tr > td").each(function () {
+$(this).text(0);
+});
+} else {
+
$("#hostedTablets").text(bigNumberForQuantity(data.details.hostedTablets));
+$("#entries").text(bigNumberForQuantity(data.details.entries));
+$("#minors").text(bigNumberForQuantity(data.details.minors));
+$("#majors").text(bigNumberForQuantity(data.details.majors));
+$("#splits").text(bigNumberForQuantity(data.details.splits));
+}
}
/**
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/server.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/server.ftl
index 5a77341fba..0c29b8ac9c 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/server.ftl
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/server.ftl
@@ -24,6 +24,9 @@
* show the table(t), length selector(l) aligned to the left and
pagination(p).
*/
$(document).ready(function() {
+// Global constant for the page
+serv = '${server}';
+
// Create a table for tserver list
tabletResults = $('#perTabletResults').DataTable({
"ajax": {
@@ -72,9 +75,9 @@
{ "data": "majorAvgES" }
]
});
-serv = '${server}';
refreshServer();
});
+
@@ -94,7 +97,15 @@
Splitting
-
+
+
+
+
+
+
+
+
+
[accumulo] branch main updated: Make rest endpoints sorted (#2649)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ab5bb96bd1 Make rest endpoints sorted (#2649)
ab5bb96bd1 is described below
commit ab5bb96bd1258b5a6720f22e855019f403dd7f1c
Author: Mike Miller
AuthorDate: Mon Apr 25 13:39:17 2022 -0400
Make rest endpoints sorted (#2649)
---
.../src/main/java/org/apache/accumulo/monitor/view/WebViews.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
index 4c1a981bb6..73b9732fd7 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
@@ -30,6 +30,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import jakarta.inject.Inject;
import jakarta.validation.constraints.Max;
@@ -463,8 +465,8 @@ public class WebViews {
return model;
}
- private List getEndpoints() {
-List endpoints = new ArrayList<>();
+ private Set getEndpoints() {
+Set endpoints = new TreeSet<>();
endpoints.add("/rest/manager");
endpoints.add("/rest/tables/namespaces");
endpoints.add("/rest/problems/summary");
@@ -488,6 +490,7 @@ public class WebViews {
endpoints.add("/rest/statistics/time/indexCacheHitRate");
endpoints.add("/rest/statistics/time/dataCacheHitRate");
endpoints.add("/rest/status");
+
return endpoints;
}
[accumulo] branch main updated: Add all endpoints page on Monitor (#2639)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e7a20c0a2b Add all endpoints page on Monitor (#2639)
e7a20c0a2b is described below
commit e7a20c0a2b038f9c8d7661ceef026bba9de2a09c
Author: Mike Miller
AuthorDate: Wed Apr 20 08:46:20 2022 -0400
Add all endpoints page on Monitor (#2639)
* Create a debug page on the monitor and an endpoint to display all
of the REST endpoints available on the monitor
* Also create a variable to store the live port number when the server
starts up
---
.../java/org/apache/accumulo/monitor/Monitor.java | 10 -
.../org/apache/accumulo/monitor/view/WebViews.java | 45 ++
.../apache/accumulo/monitor/templates/debug.ftl| 28 ++
.../apache/accumulo/monitor/templates/navbar.ftl | 5 ++-
4 files changed, 85 insertions(+), 3 deletions(-)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index d6403bc220..158119587c 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -181,6 +181,7 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
+ "'accumulo compaction-coordinator'.";
private EmbeddedWebServer server;
+ private int livePort = 0;
private ServiceLock monitorLock;
@@ -447,12 +448,13 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
int[] ports = getConfiguration().getPort(Property.MONITOR_PORT);
for (int port : ports) {
try {
-log.debug("Creating monitor on port {}", port);
+log.debug("Trying monitor on port {}", port);
server = new EmbeddedWebServer(this, port);
server.addServlet(getDefaultServlet(), "/resources/*");
server.addServlet(getRestServlet(), "/rest/*");
server.addServlet(getViewServlet(), "/*");
server.start();
+livePort = port;
break;
} catch (Exception ex) {
log.error("Unable to start embedded web server", ex);
@@ -461,6 +463,8 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
if (!server.isRunning()) {
throw new RuntimeException(
"Unable to start embedded web server on ports: " +
Arrays.toString(ports));
+} else {
+ log.debug("Monitor started on port {}", livePort);
}
try {
@@ -967,4 +971,8 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
public Optional getCoordinatorHost() {
return coordinatorHost;
}
+
+ public int getLivePort() {
+return livePort;
+ }
}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
index eb4b1e9faa..4c1a981bb6 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
@@ -446,4 +446,49 @@ public class WebViews {
return model;
}
+
+ @GET
+ @Path("all")
+ @Template(name = "/default.ftl")
+ public Map getRestView() {
+
+Map model = getModel();
+model.put("title", "Rest Endpoints");
+
+model.put("template", "debug.ftl");
+model.put("js", "functions.js");
+
+model.put("endpoints", getEndpoints());
+
+return model;
+ }
+
+ private List getEndpoints() {
+List endpoints = new ArrayList<>();
+endpoints.add("/rest/manager");
+endpoints.add("/rest/tables/namespaces");
+endpoints.add("/rest/problems/summary");
+endpoints.add("/rest/tables");
+endpoints.add("/rest/tservers");
+endpoints.add("/rest/scans");
+endpoints.add("/rest/bulkImports");
+endpoints.add("/rest/tservers/serverStats");
+endpoints.add("/rest/tservers/recovery");
+endpoints.add("/rest/logs");
+endpoints.add("/rest/problems/details");
+endpoints.add("/rest/replication");
+endpoints.add("/rest/statistics/time/ingestRate");
+endpoints.add("/rest/statistics/time/scanEntries");
+endpoints.add("/rest/statistics/time/ingestByteRate");
+endpoints.add("/rest/statistics/time/queryByteRate");
+endpoints.add("/rest/statistics/time/load");
+endpoints.add("/rest/statistics/time/lookups");
+
[accumulo] branch main updated (b00c6d216b -> 286d927956)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b00c6d216b Update Apache parent POM, modernizer (#2636) add 286d927956 Add specific timeouts to tests in SslIT (#2631) No new revisions were added by this update. Summary of changes: test/src/main/java/org/apache/accumulo/test/functional/SslIT.java | 8 1 file changed, 8 insertions(+)
[accumulo] branch main updated: Create IteratorBuilder to clean up internals (#2600)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new bd27df02c8 Create IteratorBuilder to clean up internals (#2600)
bd27df02c8 is described below
commit bd27df02c86bc224de2d39f0839bbbf8f6b46357
Author: Mike Miller
AuthorDate: Thu Apr 14 11:25:14 2022 -0400
Create IteratorBuilder to clean up internals (#2600)
* Create builder classes for iterator stack creation
* Replace IterLoad class with new IteratorBuilder and refactor classes
that use it to determine whether or not there is a bug in iterator code
---
.../core/client/ClientSideIteratorScanner.java | 16 +++---
.../core/client/admin/NewTableConfiguration.java | 4 +-
.../accumulo/core/client/rfile/RFileScanner.java | 16 +++---
.../accumulo/core/clientImpl/OfflineIterator.java | 11 ++--
.../IteratorBuilder.java} | 60 +++---
.../IteratorBuilderImpl.java} | 46 ++---
.../IteratorConfigUtil.java} | 47 +
.../IteratorConfigUtilTest.java} | 26 ++
.../accumulo/server/compaction/FileCompactor.java | 4 +-
.../accumulo/server/conf/TableConfiguration.java | 4 +-
.../accumulo/tserver/ConditionCheckerContext.java | 17 +++---
.../accumulo/tserver/tablet/ScanDataSource.java| 15 +++---
.../shell/commands/CreateTableCommand.java | 4 +-
.../test/performance/scan/CollectTabletStats.java | 11 ++--
14 files changed, 151 insertions(+), 130 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index b56d47583c..d561ce3918 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@ -33,8 +33,6 @@ import java.util.TreeSet;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.clientImpl.ScannerOptions;
-import org.apache.accumulo.core.conf.IterConfigUtil;
-import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Column;
@@ -46,6 +44,8 @@ import org.apache.accumulo.core.iterators.IteratorAdapter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder;
+import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.io.Text;
@@ -250,11 +250,13 @@ public class ClientSideIteratorScanner extends
ScannerOptions implements Scanner
SortedKeyValueIterator skvi;
try {
- IteratorEnvironment env = new
ClientSideIteratorEnvironment(getSamplerConfiguration() != null,
- getIteratorSamplerConfigurationInternal());
- IterLoad iterLoad = new
IterLoad().iters(tm.values()).iterOpts(serverSideIteratorOptions)
- .iterEnv(env).useAccumuloClassLoader(false);
- skvi = IterConfigUtil.loadIterators(smi, iterLoad);
+ IteratorEnvironment iterEnv = new ClientSideIteratorEnvironment(
+ getSamplerConfiguration() != null,
getIteratorSamplerConfigurationInternal());
+
+ IteratorBuilder ib =
+
IteratorBuilder.builder(tm.values()).opts(serverSideIteratorOptions).env(iterEnv).build();
+
+ skvi = IteratorConfigUtil.loadIterators(smi, ib);
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
index c1cb21b0c8..364a6ff122 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
@@ -38,10 +38,10 @@ import
org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.client.summary.Summarizer;
import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
import org.apache.accumulo.core.clientImpl.TableOperationsHelper;
-import org.apache.accumulo.core.conf.IterConfigUtil;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import
[accumulo-website] branch main updated: Update release notes for new tserver property (#313)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 6bc3d200 Update release notes for new tserver property (#313)
6bc3d200 is described below
commit 6bc3d2001304892c6fe13389cf9eb3a715c7ac4a
Author: Mike Miller
AuthorDate: Thu Apr 7 11:55:21 2022 -0400
Update release notes for new tserver property (#313)
---
_posts/release/2020-01-19-accumulo-2.1.0.md | 8
1 file changed, 8 insertions(+)
diff --git a/_posts/release/2020-01-19-accumulo-2.1.0.md
b/_posts/release/2020-01-19-accumulo-2.1.0.md
index b51947b5..c3f6ebed 100644
--- a/_posts/release/2020-01-19-accumulo-2.1.0.md
+++ b/_posts/release/2020-01-19-accumulo-2.1.0.md
@@ -277,6 +277,14 @@ along with the addition of a "Fetched" column indicating
when the data was retri
for External Compactions that allows seeing the progress of compactions and
many details about ongoing compactions.
{% ghi 2283 %} {% ghi 2294 %} {% ghi 2358 %}
+### New tserver scan timeout property
+
+The new property `tserver.scan.results.max.timeout` was added to allow
configuration of the timeout. A bug was
+discovered where tservers were running out of memory, partially due to this
timeout being so short. The default value
+is 1 second, but now it can be increased. It is the max time for the thrift
client handler to wait for scan results before
+timing out. {% ghi 2599 %} {% ghi 2598 %}
+
+
[accumulo] branch main updated: Rename new property for better sorting (#2604)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 26cb8ef487 Rename new property for better sorting (#2604)
26cb8ef487 is described below
commit 26cb8ef487c1a42dc39f89f011a1667938204c98
Author: Mike Miller
AuthorDate: Mon Apr 4 08:38:04 2022 -0400
Rename new property for better sorting (#2604)
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 7 ---
.../main/java/org/apache/accumulo/tserver/ThriftClientHandler.java | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index e822ea7b0c..9ca2fdc87a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -395,9 +395,6 @@ public enum Property {
"Properties in this category affect the behavior of the tablet servers",
"1.3.5"),
TSERV_CLIENT_TIMEOUT("tserver.client.timeout", "3s",
PropertyType.TIMEDURATION,
"Time to wait for clients to continue scans before closing a session.",
"1.3.5"),
- TSERV_MAX_SCAN_RESULT_TIMEOUT("tserver.max.scan.result.timeout", "1s",
PropertyType.TIMEDURATION,
- "Max time for the thrift client handler to wait for scan results before
timing out.",
- "2.1.0"),
TSERV_DEFAULT_BLOCKSIZE("tserver.default.blocksize", "1M",
PropertyType.BYTES,
"Specifies a default blocksize for the tserver caches", "1.3.5"),
TSERV_CACHE_MANAGER_IMPL("tserver.cache.manager.class",
@@ -540,6 +537,10 @@ public enum Property {
"2.0.0"),
TSERV_SCAN_EXECUTORS_META_THREADS("tserver.scan.executors.meta.threads",
"8", PropertyType.COUNT,
"The number of threads for the metadata table scan executor.", "2.0.0"),
+ TSERV_SCAN_RESULTS_MAX_TIMEOUT("tserver.scan.results.max.timeout", "1s",
+ PropertyType.TIMEDURATION,
+ "Max time for the thrift client handler to wait for scan results before
timing out.",
+ "2.1.0"),
TSERV_MIGRATE_MAXCONCURRENT("tserver.migrations.concurrent.max", "1",
PropertyType.COUNT,
"The maximum number of concurrent tablet migrations for a tablet
server", "1.3.5"),
TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s",
PropertyType.TIMEDURATION,
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
index 5264991ffd..9802e0c99c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
@@ -175,7 +175,7 @@ public class ThriftClientHandler extends
ClientServiceHandler implements TabletC
super(server.getContext(), new TransactionWatcher(server.getContext()));
this.server = server;
MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS =
server.getContext().getConfiguration()
-.getTimeInMillis(Property.TSERV_MAX_SCAN_RESULT_TIMEOUT);
+.getTimeInMillis(Property.TSERV_SCAN_RESULTS_MAX_TIMEOUT);
log.debug("{} created", ThriftClientHandler.class.getName());
}
[accumulo] branch main updated: Add new property tserver.max.scan.result.timeout (#2599)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3cf165b Add new property tserver.max.scan.result.timeout (#2599)
3cf165b is described below
commit 3cf165b4fa97514dc454eef15ee03ed1601ace9e
Author: Mike Miller
AuthorDate: Thu Mar 31 11:10:51 2022 -0400
Add new property tserver.max.scan.result.timeout (#2599)
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java| 3 +++
.../main/java/org/apache/accumulo/tserver/ThriftClientHandler.java| 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 06131bc..e822ea7 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -395,6 +395,9 @@ public enum Property {
"Properties in this category affect the behavior of the tablet servers",
"1.3.5"),
TSERV_CLIENT_TIMEOUT("tserver.client.timeout", "3s",
PropertyType.TIMEDURATION,
"Time to wait for clients to continue scans before closing a session.",
"1.3.5"),
+ TSERV_MAX_SCAN_RESULT_TIMEOUT("tserver.max.scan.result.timeout", "1s",
PropertyType.TIMEDURATION,
+ "Max time for the thrift client handler to wait for scan results before
timing out.",
+ "2.1.0"),
TSERV_DEFAULT_BLOCKSIZE("tserver.default.blocksize", "1M",
PropertyType.BYTES,
"Specifies a default blocksize for the tserver caches", "1.3.5"),
TSERV_CACHE_MANAGER_IMPL("tserver.cache.manager.class",
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
index 772014e..5264991 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java
@@ -165,7 +165,7 @@ import io.opentelemetry.context.Scope;
public class ThriftClientHandler extends ClientServiceHandler implements
TabletClientService.Iface {
private static final Logger log =
LoggerFactory.getLogger(ThriftClientHandler.class);
- private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
+ private final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS;
private static final long RECENTLY_SPLIT_MILLIES = MINUTES.toMillis(1);
private final TabletServer server;
private final WriteTracker writeTracker = new WriteTracker();
@@ -174,6 +174,8 @@ public class ThriftClientHandler extends
ClientServiceHandler implements TabletC
public ThriftClientHandler(TabletServer server) {
super(server.getContext(), new TransactionWatcher(server.getContext()));
this.server = server;
+MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS =
server.getContext().getConfiguration()
+.getTimeInMillis(Property.TSERV_MAX_SCAN_RESULT_TIMEOUT);
log.debug("{} created", ThriftClientHandler.class.getName());
}
[accumulo] branch main updated (1c0aa8f -> 4974d2c)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 1c0aa8f Bump Micrometer from 1.8.3 to 1.8.4 (#2596) add 4974d2c Fix ZooZap (#2597) No new revisions were added by this update. Summary of changes: .../base/src/main/java/org/apache/accumulo/server/util/ZooZap.java | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-)
[accumulo] branch main updated: Add missing javadoc for exceptions thrown on invalid arguments (#2594)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d093c5f Add missing javadoc for exceptions thrown on invalid
arguments (#2594)
d093c5f is described below
commit d093c5f850954b192d5bafa3bca3a931535937a8
Author: Diego Marcilio
AuthorDate: Mon Mar 28 15:28:48 2022 +0200
Add missing javadoc for exceptions thrown on invalid arguments (#2594)
* Fixing wrong javadoc for null outputStream in
SerializationUtil.serialize()
* Adding a couple @throws javadocs and tests for invalid arguments
---
.../java/org/apache/accumulo/core/conf/ClientProperty.java| 2 ++
.../apache/accumulo/core/metadata/schema/MetadataTime.java| 2 ++
.../org/apache/accumulo/core/conf/ClientPropertyTest.java | 5 +
.../accumulo/core/metadata/schema/MetadataTimeTest.java | 11 +++
.../java/org/apache/accumulo/test/util/SerializationUtil.java | 2 +-
5 files changed, 21 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
b/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
index 9f453ce..306e449 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ClientProperty.java
@@ -340,6 +340,8 @@ public enum ClientProperty {
/**
* @throws IllegalArgumentException
* if Properties does not contain all required
+ * @throws NullPointerException
+ * if {@code properties == null}
*/
public static void validate(Properties properties) {
validate(properties, true);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataTime.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataTime.java
index 0418b49..40852d4 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataTime.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataTime.java
@@ -40,6 +40,8 @@ public final class MetadataTime implements
Comparable {
* @param timestr
* string representation of a metatdata time, ex. "M12345678"
* @return a MetadataTime object represented by string
+ * @throws IllegalArgumentException
+ * if {@code timesstr == null} or {@code timestr.length() <= 1)}
*/
public static MetadataTime parse(String timestr) throws
IllegalArgumentException {
diff --git
a/core/src/test/java/org/apache/accumulo/core/conf/ClientPropertyTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/ClientPropertyTest.java
index db76b93..53ef1d1 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/ClientPropertyTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/ClientPropertyTest.java
@@ -78,4 +78,9 @@ public class ClientPropertyTest {
assertThrows(IllegalStateException.class,
() -> ClientProperty.BATCH_WRITER_LATENCY_MAX.getBytes(props));
}
+
+ @Test
+ public void validateThrowsNPEOnNullProperties() {
+assertThrows(NullPointerException.class, () ->
ClientProperty.validate(null));
+ }
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/MetadataTimeTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/MetadataTimeTest.java
index 10f2547..62a8f8a 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/MetadataTimeTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/MetadataTimeTest.java
@@ -49,6 +49,17 @@ public class MetadataTimeTest {
}
@Test
+ public void testGetInstance_nullArgument() {
+assertThrows(IllegalArgumentException.class, () ->
MetadataTime.parse(null));
+ }
+
+ @Test
+ public void testGetInstance_Invalid_timestr() {
+assertThrows(IllegalArgumentException.class, () -> MetadataTime.parse(""));
+assertThrows(IllegalArgumentException.class, () ->
MetadataTime.parse("X"));
+ }
+
+ @Test
public void testGetInstance_Millis() {
assertEquals(1234, m1234.getTime());
assertEquals(TimeType.MILLIS, m1234.getType());
diff --git
a/test/src/main/java/org/apache/accumulo/test/util/SerializationUtil.java
b/test/src/main/java/org/apache/accumulo/test/util/SerializationUtil.java
index cae2079..4660fc6 100644
--- a/test/src/main/java/org/apache/accumulo/test/util/SerializationUtil.java
+++ b/test/src/main/java/org/apache/accumulo/test/util/SerializationUtil.java
@@ -161,7 +161,7 @@ public class SerializationUtil {
* the object to serialize to bytes, may be null
* @param outputStream
* the stream to write to, must not be null
- * @throws IllegalArgumentException
+ * @throws NullPointerException
* if {@code outputStream} is {@code null}
*/
[accumulo] branch main updated: Make ZooZap keyword executable (#2591)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 69c31d4 Make ZooZap keyword executable (#2591)
69c31d4 is described below
commit 69c31d48058e8fbf121ace69324f186ab56a4e70
Author: Mike Miller
AuthorDate: Fri Mar 25 13:24:35 2022 -0400
Make ZooZap keyword executable (#2591)
---
.../java/org/apache/accumulo/server/util/ZooZap.java | 18 --
.../org/apache/accumulo/test/start/KeywordStartIT.java | 2 ++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
index e931d7a..6f6578e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.security.SecurityUtil;
+import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.KeeperException;
@@ -41,8 +42,10 @@ import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
-public class ZooZap {
+@AutoService(KeywordExecutable.class)
+public class ZooZap implements KeywordExecutable {
private static final Logger log = LoggerFactory.getLogger(ZooZap.class);
private static void message(String msg, Opts opts) {
@@ -51,6 +54,16 @@ public class ZooZap {
}
}
+ @Override
+ public String keyword() {
+return "zoo-zap";
+ }
+
+ @Override
+ public String description() {
+return "Utility for zapping Zookeeper locks";
+ }
+
static class Opts extends Help {
@Deprecated(since = "2.1.0")
@Parameter(names = "-master",
@@ -69,7 +82,8 @@ public class ZooZap {
boolean verbose = false;
}
- public static void main(String[] args) {
+ @Override
+ public void execute(String[] args) throws Exception {
Opts opts = new Opts();
opts.parseArgs(ZooZap.class.getName(), args);
diff --git
a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
index 2e6d800..7ddd46c 100644
--- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -60,6 +60,7 @@ import org.apache.accumulo.server.util.ECAdmin;
import org.apache.accumulo.server.util.Info;
import org.apache.accumulo.server.util.LoginProperties;
import org.apache.accumulo.server.util.ZooKeeperMain;
+import org.apache.accumulo.server.util.ZooZap;
import org.apache.accumulo.shell.Shell;
import org.apache.accumulo.start.Main;
import org.apache.accumulo.start.spi.KeywordExecutable;
@@ -133,6 +134,7 @@ public class KeywordStartIT {
expectSet.put("zookeeper", ZooKeeperMain.class);
expectSet.put("create-empty", CreateEmpty.class);
expectSet.put("split-large", SplitLarge.class);
+expectSet.put("zoo-zap", ZooZap.class);
Iterator>> expectIter =
expectSet.entrySet().iterator();
[accumulo-website] branch asf-site updated (3d1ede6 -> b1e00a9)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from 3d1ede6 Automatic Site Publish by Buildbot add 20270cb Automatic Site Publish by Buildbot add b1e00a9 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/contributors-guide/index.html | 2 +- output/feed.xml | 4 ++-- output/release/accumulo-2.1.0/index.html | 11 +-- output/search_data.json | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-)
[accumulo-website] branch main updated: Fix links in contributor/contributors-guide.md
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 42b5498 Fix links in contributor/contributors-guide.md
42b5498 is described below
commit 42b54986ec8af2223e698d7e93386265ee683fe0
Author: Mike Miller
AuthorDate: Fri Mar 25 12:21:08 2022 -0400
Fix links in contributor/contributors-guide.md
---
contributor/contributors-guide.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/contributor/contributors-guide.md
b/contributor/contributors-guide.md
index aefb793..1c58e07 100644
--- a/contributor/contributors-guide.md
+++ b/contributor/contributors-guide.md
@@ -71,3 +71,5 @@ For details about governance policies for the Accumulo
project view the followin
[consensus]: {{ site.baseurl }}/contributor/consensusBuilding
[lazy]: {{ site.baseurl }}/contributor/lazyConsensus
[voting]: {{ site.baseurl }}/contributor/voting
+[IntelliJ]: https://www.jetbrains.com/idea/download
+[Eclipse]: https://www.eclipse.org/downloads/packages
[accumulo] branch main updated: Use new ThreadPools method to clean up code (#2589)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7c3a978 Use new ThreadPools method to clean up code (#2589)
7c3a978 is described below
commit 7c3a9783ad01809f7c1bd5cf22471a1ffa421852
Author: Mike Miller
AuthorDate: Fri Mar 25 07:35:47 2022 -0400
Use new ThreadPools method to clean up code (#2589)
---
.../apache/accumulo/server/rpc/TServerUtils.java | 29 ++
.../accumulo/server/util/FileSystemMonitor.java| 26 +--
2 files changed, 25 insertions(+), 30 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
index b176175..ba8ba2d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
@@ -314,22 +314,19 @@ public class TServerUtils {
final ThreadPoolExecutor pool =
ThreadPools.getServerThreadPools().createFixedThreadPool(
executorThreads, threadTimeOut, TimeUnit.MILLISECONDS, serverName +
"-ClientPool", true);
// periodically adjust the number of threads we need by checking how busy
our threads are
-ThreadPools.watchCriticalScheduledTask(ThreadPools.getServerThreadPools()
-.createGeneralScheduledExecutorService(conf).scheduleWithFixedDelay(()
-> {
- // there is a minor race condition between sampling the current
state of the thread pool
- // and
- // adjusting it
- // however, this isn't really an issue, since it adjusts
periodically anyway
- if (pool.getCorePoolSize() <= pool.getActiveCount()) {
-int larger = pool.getCorePoolSize() +
Math.min(pool.getQueue().size(), 2);
-ThreadPools.resizePool(pool, () -> larger, serverName +
"-ClientPool");
- } else {
-if (pool.getCorePoolSize() > pool.getActiveCount() + 3) {
- int smaller = Math.max(executorThreads, pool.getCorePoolSize() -
1);
- ThreadPools.resizePool(pool, () -> smaller, serverName +
"-ClientPool");
-}
- }
-}, timeBetweenThreadChecks, timeBetweenThreadChecks,
TimeUnit.MILLISECONDS));
+ThreadPools.watchCriticalFixedDelay(conf, timeBetweenThreadChecks, () -> {
+ // there is a minor race condition between sampling the current state of
the thread pool
+ // and adjusting it however, this isn't really an issue, since it
adjusts periodically
+ if (pool.getCorePoolSize() <= pool.getActiveCount()) {
+int larger = pool.getCorePoolSize() + Math.min(pool.getQueue().size(),
2);
+ThreadPools.resizePool(pool, () -> larger, serverName + "-ClientPool");
+ } else {
+if (pool.getCorePoolSize() > pool.getActiveCount() + 3) {
+ int smaller = Math.max(executorThreads, pool.getCorePoolSize() - 1);
+ ThreadPools.resizePool(pool, () -> smaller, serverName +
"-ClientPool");
+}
+ }
+});
return pool;
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
index 255f453..1bcc77c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
@@ -29,7 +29,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -117,20 +116,19 @@ public class FileSystemMonitor {
// Create a task to check each mount periodically to see if its state has
changed.
for (Mount mount : mounts) {
- ThreadPools.watchCriticalScheduledTask(ThreadPools.getServerThreadPools()
- .createGeneralScheduledExecutorService(conf).scheduleWithFixedDelay(
- Threads.createNamedRunnable(mount.mountPoint + "filesystem
monitor", () -> {
-try {
- checkMount(mount);
-} catch (final Exception e) {
- Halt.halt(-42, new Runnable() {
-@Override
-public void run() {
- log.error("Exception while checking mount points,
halting process", e);
-}
- });
+ ThreadPools.watchCriticalFixedDelay(conf, period,
+ Threads.createNamedRunnable(mount.mountPoint + "filesystem monitor",
() -> {
+try {
+
[accumulo-website] branch asf-site updated (d1d5d88 -> 3d1ede6)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from d1d5d88 Automatic Site Publish by Buildbot add 3d1ede6 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 ++-- output/release/accumulo-2.1.0/index.html | 6 ++ output/search_data.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-)
[accumulo-website] branch main updated: Update release notes for consistency check thread (#311)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 3fcf2c6 Update release notes for consistency check thread (#311)
3fcf2c6 is described below
commit 3fcf2c612627c456b8f1f95a64ac89228d247898
Author: Mike Miller
AuthorDate: Thu Mar 24 13:35:01 2022 -0400
Update release notes for consistency check thread (#311)
Co-authored-by: Dom G.
---
_posts/release/2020-01-19-accumulo-2.1.0.md | 6 ++
1 file changed, 6 insertions(+)
diff --git a/_posts/release/2020-01-19-accumulo-2.1.0.md
b/_posts/release/2020-01-19-accumulo-2.1.0.md
index b1152db..30f6d53 100644
--- a/_posts/release/2020-01-19-accumulo-2.1.0.md
+++ b/_posts/release/2020-01-19-accumulo-2.1.0.md
@@ -230,6 +230,12 @@ failsafe to prevent a single ingested file from being
distributed across too muc
by the new bulk import technique and is the maximum number of tablets allowed
for one bulk import file. When this property
is set, an error will be thrown when the value is exceeded during a bulk
import. {% ghi 1614 %}
+### New Health check Thread in tserver
+
+A new thread was added to the tablet server to periodically verify tablet
metadata. {% ghi 2320 %} This thread also prints
+to the debug log how long it takes the tserver to scan the metadata table. The
property tserver.health.check.interval was
+added to control the frequency at which this health check takes place. {% ghi
2583 %}
+
### New ability for user to define context classloaders
Deprecated the existing VFS ClassLoader for eventual removal and created a new
mechanism for users
[accumulo] branch main updated: Add trace and debug log to consistency check (#2583)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new dd81d60 Add trace and debug log to consistency check (#2583)
dd81d60 is described below
commit dd81d608bff88835676e756ffcb2a2f6b69c9be8
Author: Mike Miller
AuthorDate: Thu Mar 24 10:05:44 2022 -0400
Add trace and debug log to consistency check (#2583)
* Closes #2577
* Add trace span and time measurement around consistency check
so we get an idea of how long metadata scans are taking
* Create new property tserver.health.check.interval to make it configurable
* Create new method watchCriticalFixedDelay() in ThreadPools
Co-authored-by: Keith Turner
---
.../org/apache/accumulo/core/conf/Property.java| 2 +
.../accumulo/core/util/threads/ThreadPools.java| 7 ++
.../org/apache/accumulo/tserver/TabletServer.java | 80 +-
3 files changed, 57 insertions(+), 32 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 667d78b..06131bc 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -656,6 +656,8 @@ public enum Property {
"1.4.0"),
TSERV_BULK_TIMEOUT("tserver.bulk.timeout", "5m", PropertyType.TIMEDURATION,
"The time to wait for a tablet server to process a bulk import
request.", "1.4.3"),
+ TSERV_HEALTH_CHECK_FREQ("tserver.health.check.interval", "30m",
PropertyType.TIMEDURATION,
+ "The time between tablet server health checks.", "2.1.0"),
TSERV_MINTHREADS("tserver.server.threads.minimum", "20", PropertyType.COUNT,
"The minimum number of threads to use to handle incoming requests.",
"1.4.0"),
TSERV_MINTHREADS_TIMEOUT("tserver.server.threads.timeout", "0s",
PropertyType.TIMEDURATION,
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
index 73cb308..548852a 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
@@ -166,6 +166,13 @@ public class ThreadPools {
CRITICAL_RUNNING_TASKS.add(future);
}
+ public static void watchCriticalFixedDelay(AccumuloConfiguration aconf, long
intervalMillis,
+ Runnable runnable) {
+ScheduledFuture future =
getServerThreadPools().createGeneralScheduledExecutorService(aconf)
+.scheduleWithFixedDelay(runnable, intervalMillis, intervalMillis,
TimeUnit.MILLISECONDS);
+CRITICAL_RUNNING_TASKS.add(future);
+ }
+
public static void watchNonCriticalScheduledTask(ScheduledFuture future) {
NON_CRITICAL_RUNNING_TASKS.add(future);
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 3de8187..8c33967 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -23,12 +23,17 @@ import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
+import static
org.apache.accumulo.core.util.threads.ThreadPools.watchCriticalFixedDelay;
+import static
org.apache.accumulo.core.util.threads.ThreadPools.watchCriticalScheduledTask;
+import static
org.apache.accumulo.core.util.threads.ThreadPools.watchNonCriticalScheduledTask;
import static
org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.UnknownHostException;
import java.security.SecureRandom;
+import java.time.Duration;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -164,6 +169,9 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.context.Scope;
+
public class TabletServer extends AbstractServer {
private static final SecureRandom random = new SecureRandom();
@@ -287,7 +295,7 @@ public clas
[accumulo] branch main updated: Create ListBulkCommandTest (#2582)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 738b1c1 Create ListBulkCommandTest (#2582)
738b1c1 is described below
commit 738b1c15b6f6383f6d1609a1cba702cd350f0a42
Author: Mike Miller
AuthorDate: Wed Mar 23 09:09:35 2022 -0400
Create ListBulkCommandTest (#2582)
---
.../shell/commands/BulkImportListIterator.java | 4 +-
.../shell/commands/ListBulkCommandTest.java| 112 +
2 files changed, 114 insertions(+), 2 deletions(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java
index 1c8bfe4..0607988 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/BulkImportListIterator.java
@@ -33,11 +33,11 @@ public class BulkImportListIterator implements
Iterator {
public BulkImportListIterator(List tservers, ManagerMonitorInfo
stats) {
List result = new ArrayList<>();
-for (BulkImportStatus status : stats.bulkImports) {
+for (BulkImportStatus status : stats.getBulkImports()) {
result.add(format(status));
}
if (!tservers.isEmpty()) {
- for (TabletServerStatus tserver : stats.tServerInfo) {
+ for (TabletServerStatus tserver : stats.getTServerInfo()) {
if (tservers.contains(tserver.name)) {
result.add(tserver.name + ":");
for (BulkImportStatus status : tserver.bulkImports) {
diff --git
a/shell/src/test/java/org/apache/accumulo/shell/commands/ListBulkCommandTest.java
b/shell/src/test/java/org/apache/accumulo/shell/commands/ListBulkCommandTest.java
new file mode 100644
index 000..1dccfc9
--- /dev/null
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ListBulkCommandTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo;
+import org.apache.accumulo.core.master.thrift.BulkImportState;
+import org.apache.accumulo.core.master.thrift.BulkImportStatus;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.junit.jupiter.api.Test;
+
+public class ListBulkCommandTest {
+ private static final List tservers = List.of("tserver1", "tserver2");
+
+ /**
+ * Test the iterator used by BulkCommand prints correctly
+ */
+ @Test
+ public void testBulkImportListIterator() {
+ManagerMonitorInfo mmi = createMock(ManagerMonitorInfo.class);
+List statusList = new ArrayList<>();
+List bulkImports = new ArrayList<>();
+
+// tserver 1
+var bis1 = new BulkImportStatus();
+bis1.filename = "file1";
+bis1.startTime = System.currentTimeMillis();
+bis1.state = BulkImportState.COPY_FILES;
+bulkImports.add(bis1);
+var ts1 = new TabletServerStatus();
+ts1.name = "tserver1";
+ts1.bulkImports = new ArrayList<>(bulkImports);
+bulkImports.clear();
+
+// tserver 2
+var bis2 = new BulkImportStatus();
+bis2.filename = "file2";
+bis2.startTime = System.currentTimeMillis();
+bis2.state = BulkImportState.LOADING;
+bulkImports.add(bis2);
+var ts2 = new TabletServerStatus();
+ts2.name = "tserver2";
+ts2.bulkImports = new ArrayList<>(bulkImports);
+bulkImports.clear();
+
+statusList.add(ts1);
+statusList.add(ts2);
+
+bulkImports.add(bis1);
+bulkImports.add(bis2);
+
[accumulo] branch main updated: Rename Duration to DurationFormat (#2579)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 26ee87e Rename Duration to DurationFormat (#2579)
26ee87e is described below
commit 26ee87e6567e24d0de88879f38ec3ecf2c85e0ce
Author: Mike Miller
AuthorDate: Mon Mar 21 09:13:38 2022 -0400
Rename Duration to DurationFormat (#2579)
* Rename to DurationFormat to prevent confusion with java Duration
* Also clean up the class
---
.../util/{Duration.java => DurationFormat.java}| 64 +++---
.../shell/commands/ActiveCompactionHelper.java | 11 ++--
.../shell/commands/ActiveScanIterator.java | 9 +--
.../shell/commands/BulkImportListIterator.java | 6 +-
4 files changed, 58 insertions(+), 32 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/util/Duration.java
b/core/src/main/java/org/apache/accumulo/core/util/DurationFormat.java
similarity index 58%
rename from core/src/main/java/org/apache/accumulo/core/util/Duration.java
rename to core/src/main/java/org/apache/accumulo/core/util/DurationFormat.java
index 7541e33..1cdc6bb 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/Duration.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/DurationFormat.java
@@ -18,40 +18,64 @@
*/
package org.apache.accumulo.core.util;
-public class Duration {
+public class DurationFormat {
+ private final String str;
- public static String format(long time) {
-return format(time, "", "");
- }
-
- public static String format(long time, String space, String zero) {
+ public DurationFormat(long time, String space) {
+String dash = "-";
long ms, sec, min, hr, day, yr;
-ms = sec = min = hr = day = yr = -1;
-if (time == 0)
- return zero;
+
+if (time == 0) {
+ str = dash;
+ return;
+}
+
ms = time % 1000;
time /= 1000;
-if (time == 0)
- return String.format("%dms", ms);
+
+if (time == 0) {
+ str = String.format("%dms", ms);
+ return;
+}
+
sec = time % 60;
time /= 60;
-if (time == 0)
- return String.format("%ds" + space + "%dms", sec, ms);
+
+if (time == 0) {
+ str = String.format("%ds" + space + "%dms", sec, ms);
+ return;
+}
+
min = time % 60;
time /= 60;
-if (time == 0)
- return String.format("%dm" + space + "%ds", min, sec);
+
+if (time == 0) {
+ str = String.format("%dm" + space + "%ds", min, sec);
+ return;
+}
+
hr = time % 24;
time /= 24;
-if (time == 0)
- return String.format("%dh" + space + "%dm", hr, min);
+
+if (time == 0) {
+ str = String.format("%dh" + space + "%dm", hr, min);
+ return;
+}
+
day = time % 365;
time /= 365;
-if (time == 0)
- return String.format("%dd" + space + "%dh", day, hr);
+
+if (time == 0) {
+ str = String.format("%dd" + space + "%dh", day, hr);
+ return;
+}
yr = time;
-return String.format("%dy" + space + "%dd", yr, day);
+str = String.format("%dy" + space + "%dd", yr, day);
}
+ @Override
+ public String toString() {
+return str;
+ }
}
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ActiveCompactionHelper.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ActiveCompactionHelper.java
index 1d08abf..9b1befe 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/ActiveCompactionHelper.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/ActiveCompactionHelper.java
@@ -31,7 +31,7 @@ import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.ActiveCompaction;
import org.apache.accumulo.core.client.admin.InstanceOperations;
-import org.apache.accumulo.core.util.Duration;
+import org.apache.accumulo.core.util.DurationFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,11 +87,12 @@ class ActiveCompactionHelper {
String host = ac.getHost().getAddress() + ":" + ac.getHost().getPort() +
hostSuffix;
try {
+ var dur = new DurationFormat(ac.getAge(), "");
return String.format(
- "%21s | %9s | %5s | %6s | %5s | %5s | %15s | %-40s | %5s | %35s |
%9s | %s", host,
- Duration.format(ac.getAge(), "", "-"), ac.getType(), ac.getReason(),
- shortenCount(ac.getEntriesRead()),
shortenCount(ac.getEntriesWritten()), ac.getTable(),
- ac.getTablet(), ac.get
[accumulo] branch main updated (a9bd3a1 -> a4e4f19)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from a9bd3a1 Remove unneeded VisibleForTesting TabletServer stuff (#2576) add a4e4f19 Fix typo in DatafileManager No new revisions were added by this update. Summary of changes: .../main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo-testing] branch main updated: Fixup testing for changes in Accumulo regarding ThreadPools (#197)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new da4a434 Fixup testing for changes in Accumulo regarding ThreadPools
(#197)
da4a434 is described below
commit da4a4343d738f4861d7109290d3618f0a09c93b8
Author: Dave Marion
AuthorDate: Thu Mar 17 13:03:43 2022 -0400
Fixup testing for changes in Accumulo regarding ThreadPools (#197)
---
src/main/java/org/apache/accumulo/testing/randomwalk/Module.java | 3 ++-
src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/Module.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/Module.java
index 96ba004..8e85e20 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/Module.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/Module.java
@@ -222,7 +222,8 @@ public class Module extends Node {
fixture.setUp(state, env);
}
-ExecutorService service = ThreadPools.createFixedThreadPool(1, "RandomWalk
Runner", false);
+ExecutorService service =
ThreadPools.getServerThreadPools().createFixedThreadPool(1,
+"RandomWalk Runner", false);
try {
Node initNode = getNode(initNodeId);
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java
index b39cf7f..ebeb72d 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java
@@ -60,8 +60,8 @@ public class Setup extends Test {
state.set("fs", FileSystem.get(env.getHadoopConfiguration()));
state.set("bulkImportSuccess", "true");
BulkPlusOne.counter.set(0l);
-ThreadPoolExecutor e = ThreadPools.createFixedThreadPool(MAX_POOL_SIZE,
"bulkImportPool",
-false);
+ThreadPoolExecutor e =
ThreadPools.getServerThreadPools().createFixedThreadPool(MAX_POOL_SIZE,
+"bulkImportPool", false);
state.set("pool", e);
}
[accumulo] branch main updated (dca3ae8 -> 8acc4ca)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from dca3ae8 Set default UncaughtExceptionHandler in client, enable override (#2554) add 8acc4ca Improve log message for Merges (#2563) No new revisions were added by this update. Summary of changes: server/manager/pom.xml| 4 .../java/org/apache/accumulo/manager/FateServiceHandler.java | 11 --- 2 files changed, 12 insertions(+), 3 deletions(-)
[accumulo] branch main updated (05a741a -> cd9b536)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 05a741a Sync FateIT.getTxStatus (#2550) add cd9b536 Refactor FateCommand and create FateCommandTest (#2541) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/fate/AdminUtil.java | 36 ++--- .../apache/accumulo/manager/util/FateAdmin.java| 9 +- .../accumulo/shell/commands/FateCommand.java | 177 +++-- .../accumulo/shell/commands/FateCommandTest.java | 109 + .../test/functional/FateConcurrencyIT.java | 11 +- .../test/functional/FunctionalTestUtils.java | 5 +- 6 files changed, 235 insertions(+), 112 deletions(-) create mode 100644 shell/src/test/java/org/apache/accumulo/shell/commands/FateCommandTest.java
[accumulo-testing] branch main updated: Make MulitTable reuse tables (#191)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new d23c2fc Make MulitTable reuse tables (#191)
d23c2fc is described below
commit d23c2fcbe47ae72df9f667235b6b7f851882a184
Author: Mike Miller
AuthorDate: Fri Feb 18 11:47:29 2022 -0500
Make MulitTable reuse tables (#191)
* This allows restarting rw tests and allow using of the same tables
across multiple threads
* Other various improvements to MultiTable
---
.../testing/randomwalk/multitable/BulkImport.java | 47 +-
.../testing/randomwalk/multitable/Commit.java | 18 -
.../testing/randomwalk/multitable/CopyTable.java | 7
.../randomwalk/multitable/MultiTableFixture.java | 26 ++--
.../testing/randomwalk/multitable/Write.java | 28 +
.../resources/randomwalk/modules/MultiTable.xml| 1 -
6 files changed, 95 insertions(+), 32 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
index 4c43b0a..b2a5c44 100644
---
a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
+++
b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/BulkImport.java
@@ -25,6 +25,7 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.accumulo.core.client.IteratorSetting.Column;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.rfile.RFileWriter;
import org.apache.accumulo.core.data.Key;
@@ -39,7 +40,7 @@ import org.apache.hadoop.io.Text;
public class BulkImport extends Test {
- public static final int LOTS = 10;
+ public static final int ROWS = 1_000_000;
public static final int COLS = 10;
public static final List COLNAMES = new ArrayList<>();
public static final Text CHECK_COLUMN_FAMILY = new Text("cf");
@@ -61,7 +62,7 @@ public class BulkImport extends Test {
List tables = (List) state.get("tableList");
if (tables.isEmpty()) {
- log.debug("No tables to ingest into");
+ log.trace("No tables to ingest into");
return;
}
@@ -77,8 +78,8 @@ public class BulkImport extends Test {
final boolean useLegacyBulk = rand.nextBoolean();
TreeSet rows = new TreeSet<>();
-for (int i = 0; i < LOTS; i++)
- rows.add(uuid + String.format("__%05d", i));
+for (int i = 0; i < ROWS; i++)
+ rows.add(uuid + String.format("__%06d", i));
String markerColumnQualifier = String.format("%07d",
counter.incrementAndGet());
log.debug("Preparing {} bulk import to {}", useLegacyBulk ? "legacy" :
"new", tableName);
@@ -96,24 +97,30 @@ public class BulkImport extends Test {
}
f.close();
}
-if (useLegacyBulk) {
- env.getAccumuloClient().tableOperations().importDirectory(tableName,
dir.toString(),
- fail.toString(), true);
- FileStatus[] failures = fs.listStatus(fail);
- if (failures != null && failures.length > 0) {
-state.set("bulkImportSuccess", "false");
-throw new Exception(failures.length + " failure files found importing
files from " + dir);
+log.debug("Starting {} bulk import to {}", useLegacyBulk ? "legacy" :
"new", tableName);
+try {
+ if (useLegacyBulk) {
+env.getAccumuloClient().tableOperations().importDirectory(tableName,
dir.toString(),
+fail.toString(), true);
+FileStatus[] failures = fs.listStatus(fail);
+if (failures != null && failures.length > 0) {
+ state.set("bulkImportSuccess", "false");
+ throw new Exception(failures.length + " failure files found
importing files from " + dir);
+}
+ } else {
+
env.getAccumuloClient().tableOperations().importDirectory(dir.toString()).to(tableName)
+.tableTime(true).load();
}
-} else {
-
env.getAccumuloClient().tableOperations().importDirectory(dir.toString()).to(tableName)
- .tableTime(true).load();
-}
-fs.delete(dir, true);
-fs.delete(fail, true);
-log.debug("Finished {} bulk import to {} start: {} last: {} marker: {}",
-useLegacyBulk ? "legacy" : "new", tableName, rows.first(), rows.last(),
-markerColumnQualifier);
+ fs.delete(dir, true);
+ fs.delete(fail, true);
+ log.debug("Finished {} bulk import to {} start: {} last: {} marker: {}",
+
[accumulo] branch main updated: Refactor Shell ITs (#2503)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8b8a6f7 Refactor Shell ITs (#2503)
8b8a6f7 is described below
commit 8b8a6f7339fdcfbf4118c6164eb47035d857f0d9
Author: Mike Miller
AuthorDate: Fri Feb 18 04:49:10 2022 -0500
Refactor Shell ITs (#2503)
* Create ShellCreateTableIT from tests moved from ShellServerIT to help
with debugging
* Create MockShell class
* Modify ShellServerIT to not delete tables after ever test
* Move ShellConfigIT into shell package
* Move ShellIT into shell package
---
.../org/apache/accumulo/test/VolumeChooserIT.java | 2 +-
.../accumulo/test/shell/ErrorMessageCallback.java | 48 ++
.../org/apache/accumulo/test/shell/MockShell.java | 185 +
.../accumulo/test/{ => shell}/ShellConfigIT.java | 10 +-
.../accumulo/test/shell/ShellCreateTableIT.java| 713 ++
.../apache/accumulo/test/{ => shell}/ShellIT.java | 2 +-
.../accumulo/test/{ => shell}/ShellServerIT.java | 827 +
7 files changed, 957 insertions(+), 830 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
index bb474ea..6419c6c 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeChooserIT.java
@@ -65,7 +65,7 @@ public class VolumeChooserIT extends ConfigurableMacBase {
private static final String TP =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey();
static final String PREFERRED_CHOOSER_PROP = TP + "volume.preferred";
- static final String PERTABLE_CHOOSER_PROP = TP + "volume.chooser";
+ public static final String PERTABLE_CHOOSER_PROP = TP + "volume.chooser";
private static final String GP =
Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey();
diff --git
a/test/src/main/java/org/apache/accumulo/test/shell/ErrorMessageCallback.java
b/test/src/main/java/org/apache/accumulo/test/shell/ErrorMessageCallback.java
new file mode 100644
index 000..98ac0a1
--- /dev/null
+++
b/test/src/main/java/org/apache/accumulo/test/shell/ErrorMessageCallback.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.shell;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+
+public class ErrorMessageCallback {
+ private final String errorMessage;
+
+ public ErrorMessageCallback(Properties properties) {
+errorMessage = checkAuths(properties);
+ }
+
+ public ErrorMessageCallback() {
+errorMessage = "";
+ }
+
+ private String checkAuths(Properties properties) {
+try (AccumuloClient c = Accumulo.newClient().from(properties).build()) {
+ return "Current auths for root are: " +
c.securityOperations().getUserAuthorizations("root");
+} catch (Exception e) {
+ return "Could not check authorizations";
+}
+ }
+
+ public String getErrorMessage() {
+return errorMessage;
+ }
+}
diff --git a/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
b/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
new file mode 100644
index 000..b7ae57e
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/shell/MockShell.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software
[accumulo] branch main updated (40fb5d0 -> f8ec16c)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from 40fb5d0 Make build work on Java 17 (#2500)
add f8ec16c Move Test jars and update their ITs (#2499)
No new revisions were added by this update.
Summary of changes:
.../apache/accumulo/harness/AccumuloITBase.java| 16 +++
.../org/apache/accumulo/test/ShellServerIT.java| 31 +
.../test/compaction/UserCompactionStrategyIT.java | 17 ++-
.../accumulo/test/functional/ClassLoaderIT.java| 4 +--
.../test/functional/ConfigurableCompactionIT.java | 17 ++-
.../apache/accumulo/test}/FooConstraint.jar| Bin
.../apache/accumulo/test}/FooConstraint_2_1.jar| Bin
.../{ => org/apache/accumulo/test}/FooFilter.jar | Bin
.../accumulo/test}/ShellServerIT-iterators.jar | Bin
.../apache/accumulo/test}/TestCombinerX.jar| Bin
.../apache/accumulo/test}/TestCombinerY.jar| Bin
.../apache/accumulo/test}/TestCompactionStrat.jar | Bin
12 files changed, 29 insertions(+), 56 deletions(-)
rename test/src/main/resources/{ =>
org/apache/accumulo/test}/FooConstraint.jar (100%)
rename test/src/main/resources/{ =>
org/apache/accumulo/test}/FooConstraint_2_1.jar (100%)
rename test/src/main/resources/{ => org/apache/accumulo/test}/FooFilter.jar
(100%)
rename test/src/main/resources/{ =>
org/apache/accumulo/test}/ShellServerIT-iterators.jar (100%)
rename test/src/main/resources/{ =>
org/apache/accumulo/test}/TestCombinerX.jar (100%)
rename test/src/main/resources/{ =>
org/apache/accumulo/test}/TestCombinerY.jar (100%)
rename test/src/main/resources/{ =>
org/apache/accumulo/test}/TestCompactionStrat.jar (100%)
[accumulo] branch main updated: Fix Fate goal message
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new afe4791 Fix Fate goal message
afe4791 is described below
commit afe4791a673ee039138d5f8e8a8f608dea19f57e
Author: Mike Miller
AuthorDate: Tue Feb 15 08:04:21 2022 -0500
Fix Fate goal message
---
core/src/main/java/org/apache/accumulo/fate/Fate.java | 2 +-
.../src/main/java/org/apache/accumulo/manager/FateServiceHandler.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/fate/Fate.java
b/core/src/main/java/org/apache/accumulo/fate/Fate.java
index 769bf77..a2559ea 100644
--- a/core/src/main/java/org/apache/accumulo/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/fate/Fate.java
@@ -274,7 +274,7 @@ public class Fate {
if (store.getStatus(tid) == TStatus.NEW) {
if (store.top(tid) == null) {
try {
-log.info("Seeding {} goal: {}", FateTxId.formatTid(tid),
goalMessage);
+log.info("Seeding {} {}", FateTxId.formatTid(tid), goalMessage);
store.push(tid, repo);
} catch (StackOverflowException e) {
// this should not happen
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
index 1dbd5b6..2f2da48 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
@@ -113,7 +113,7 @@ class FateServiceHandler implements FateService.Iface {
List arguments, Map options, boolean
autoCleanup)
throws ThriftSecurityException, ThriftTableOperationException {
authenticate(c);
-String goalMessage = "Op: " + op.toString() + " ";
+String goalMessage = op.toString() + " ";
switch (op) {
case NAMESPACE_CREATE: {
