[accumulo-testing] branch main updated: Update pom to use accumulo 3.0.0-SNAPSHOT instead of 2.1.0 (#256)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new dc42aee Update pom to use accumulo 3.0.0-SNAPSHOT instead of 2.1.0
(#256)
dc42aee is described below
commit dc42aeec4b4bbe141a641c8fcc01af4c6ef42991
Author: Jeffrey Manno
AuthorDate: Sun Nov 20 12:04:00 2022 -0500
Update pom to use accumulo 3.0.0-SNAPSHOT instead of 2.1.0 (#256)
---
pom.xml | 2 +-
.../apache/accumulo/testing/randomwalk/shard/SortTool.java| 11 ---
2 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/pom.xml b/pom.xml
index fcc758c..2e4d6dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
Apache Accumulo Testing
Testing tools for Apache Accumulo
-2.1.0-SNAPSHOT
+3.0.0-SNAPSHOT
${project.basedir}/contrib/Eclipse-Accumulo-Codestyle.xml
11
11
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
index d329f51..5e57e5d 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/shard/SortTool.java
@@ -45,7 +45,6 @@ public class SortTool extends Configured implements Tool {
this.splits = splits;
}
- @SuppressWarnings("deprecation")
@Override
public int run(String[] args) throws Exception {
Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
@@ -60,18 +59,16 @@ public class SortTool extends Configured implements Tool {
SequenceFileInputFormat.setInputPaths(job, seqFile);
job.setPartitionerClass(
-
org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner.class);
-
org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner.setSplitFile(job,
-splitFile);
+
org.apache.accumulo.hadoop.mapreduce.partition.KeyRangePartitioner.class);
+
org.apache.accumulo.hadoop.mapreduce.partition.KeyRangePartitioner.setSplitFile(job,
splitFile);
job.setMapOutputKeyClass(Key.class);
job.setMapOutputValueClass(Value.class);
job.setNumReduceTasks(splits.size() + 1);
-job.setOutputFormatClass(
-
org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat.class);
-
org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat.setOutputPath(job,
+
job.setOutputFormatClass(org.apache.accumulo.hadoop.mapreduce.AccumuloFileOutputFormat.class);
+
org.apache.accumulo.hadoop.mapreduce.AccumuloFileOutputFormat.setOutputPath(job,
new Path(outputDir));
job.waitForCompletion(true);
[accumulo-testing] branch main updated: Refactor logging in Verify for RW Bulk (#254)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 44e1377 Refactor logging in Verify for RW Bulk (#254)
44e1377 is described below
commit 44e1377ccd79802d495c9cdca5278a4e2c7a9dd1
Author: Jeffrey Manno
AuthorDate: Sun Nov 20 09:29:24 2022 -0500
Refactor logging in Verify for RW Bulk (#254)
* Refactor logging in Verify for RW Bulk to give more useful information to
the user.
---
.../accumulo/testing/randomwalk/bulk/Verify.java | 30 +++---
1 file changed, 21 insertions(+), 9 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
index d39ae10..8d61149 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
@@ -67,12 +67,14 @@ public class Verify extends Test {
String user = env.getAccumuloClient().whoami();
Authorizations auths =
env.getAccumuloClient().securityOperations().getUserAuthorizations(user);
RowIterator rowIter;
+boolean errorFound = false;
try (Scanner scanner =
env.getAccumuloClient().createScanner(Setup.getTableName(), auths)) {
scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
for (Entry entry : scanner) {
byte[] value = entry.getValue().get();
if (!Arrays.equals(value, zero)) {
- throw new Exception("Bad key at " + entry);
+ log.error("Bad key found at {}", entry);
+ errorFound = true;
}
}
@@ -87,26 +89,36 @@ public class Verify extends Test {
while (row.hasNext()) {
Entry entry = row.next();
- if (rowText == null)
+ if (rowText == null) {
rowText = entry.getKey().getRow();
+ }
long curr =
Long.parseLong(entry.getKey().getColumnQualifier().toString());
- if (curr - 1 != prev)
-throw new Exception(
-"Bad marker count " + entry.getKey() + " " + entry.getValue()
+ " " + prev);
+ if (curr - 1 != prev) {
+log.error("Bad market count. Current row: {} {}, Previous row
marker: {}",
+entry.getKey(), entry.getValue(), prev);
+errorFound = true;
+ }
- if (!entry.getValue().toString().equals("1"))
-throw new Exception("Bad marker value " + entry.getKey() + " " +
entry.getValue());
+ if (!entry.getValue().toString().equals("1")) {
+log.error("Bad marker value for row {} {}.\n Value expected to be
one", entry.getKey(),
+entry.getValue());
+errorFound = true;
+ }
prev = curr;
}
if (BulkPlusOne.counter.get() != prev) {
- throw new Exception("Row " + rowText + " does not have all markers "
- + BulkPlusOne.counter.get() + " " + prev);
+ log.error("Row {} does not have all markers. Current marker: {},
Previous marker:{}",
+ rowText, BulkPlusOne.counter.get(), prev);
+ errorFound = true;
}
}
+ if (errorFound) {
+throw new Exception("Error found during Verify");
+ }
}
log.info("Test successful on table " + Setup.getTableName());
[accumulo-testing] branch main updated: Add maxdepth parameter so that unit modules are not displayed in rwalk(#218)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new b46779c Add maxdepth parameter so that unit modules are not displayed in rwalk(#218) b46779c is described below commit b46779c225193f6f72d753cb303c72949358c551 Author: Dom G AuthorDate: Sat Sep 10 13:43:25 2022 -0400 Add maxdepth parameter so that unit modules are not displayed in rwalk(#218) --- bin/rwalk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rwalk b/bin/rwalk index 42009d3..002b2e2 100755 --- a/bin/rwalk +++ b/bin/rwalk @@ -28,7 +28,7 @@ Usage: $0 () Modules listed below located in /src/main/resources/randomwalk/modules EOF - find "$at_home/src/main/resources/randomwalk/modules/." -name "*.xml" -printf "%f\n" + find "$at_home/src/main/resources/randomwalk/modules/." -maxdepth 1 -name "*.xml" -printf "%f\n" } export CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
[accumulo-testing] branch main updated: Fix YieldingScanExecutorPT by adding shaded jar to classpath (#175)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 97d0441 Fix YieldingScanExecutorPT by adding shaded jar to classpath
(#175)
97d0441 is described below
commit 97d0441bc07d55a76b380a8be6102a4758bf23b7
Author: Jeffrey Manno
AuthorDate: Sun Jul 17 12:59:10 2022 -0400
Fix YieldingScanExecutorPT by adding shaded jar to classpath (#175)
* add shaded jar to class path to fix PT
---
bin/performance | 8 ++--
pom.xml | 6 ++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/bin/performance b/bin/performance
index e69603a..1834f91 100755
--- a/bin/performance
+++ b/bin/performance
@@ -59,8 +59,12 @@ if [ ! -f "$at_home/conf/cluster-control.sh" ]; then
fi
source "$at_home"/conf/cluster-control.sh
-build_shade_jar
-CP="$at_home/target/accumulo-testing-shaded.jar:$(get_hadoop_client)"
+export TEST_JAR_PATH="${at_home}/target/accumulo-testing-shaded.jar"
+if [[ ! -f "$TEST_JAR_PATH" ]]; then
+ build_shade_jar
+fi
+CP="$at_shaded_jar:$(get_hadoop_client)"
+export CLASSPATH="$at_shaded_jar:$CLASSPATH"
perf_pkg="org.apache.accumulo.testing.performance.impl"
case "$1" in
run)
diff --git a/pom.xml b/pom.xml
index 257f0dc..9b576b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -354,6 +354,12 @@
META-INF/*.SF
META-INF/*.DSA
META-INF/*.RSA
+META-INF/*.txt
+META-INF/*.MF
+META-INF/DEPENDENCIES
+META-INF/LICENSE
+META-INF/NOTICE
+
META-INF/io.netty.versions.properties
[accumulo] branch main updated: Add consistency checks for lastFlushID and lastCompactID (#2397)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 24f16e5 Add consistency checks for lastFlushID and lastCompactID
(#2397)
24f16e5 is described below
commit 24f16e5c0279fcc2e0bc1ff9c17fce791f7437f6
Author: Luke Foster <84727868+foste...@users.noreply.github.com>
AuthorDate: Sat Feb 19 09:52:21 2022 -0500
Add consistency checks for lastFlushID and lastCompactID (#2397)
---
.../org/apache/accumulo/tserver/tablet/Tablet.java | 20
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 56645d6..3fb330e 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1378,7 +1378,7 @@ public class Tablet {
try {
var tabletMeta = context.getAmple().readTablet(extent, ColumnType.FILES,
ColumnType.LOGS,
- ColumnType.ECOMP, ColumnType.PREV_ROW);
+ ColumnType.ECOMP, ColumnType.PREV_ROW, ColumnType.FLUSH_ID,
ColumnType.COMPACT_ID);
if (tabletMeta == null) {
String msg = "Closed tablet " + extent + " not found in metadata";
@@ -1402,6 +1402,20 @@ public class Tablet {
throw new RuntimeException(msg);
}
+ if (tabletMeta.getFlushId().orElse(-1) != lastFlushID) {
+String msg = "Closed tablet " + extent + " lastFlushID is inconsistent
with metadata : "
++ tabletMeta.getFlushId().orElse(-1) + " != " + lastFlushID;
+log.error(msg);
+throw new RuntimeException(msg);
+ }
+
+ if (tabletMeta.getCompactId().orElse(-1) != lastCompactID) {
+String msg = "Closed tablet " + extent + " lastCompactID is
inconsistent with metadata : "
++ tabletMeta.getCompactId().orElse(-1) + " != " + lastCompactID;
+log.error(msg);
+throw new RuntimeException(msg);
+ }
+
compareToDataInMemory(tabletMeta);
} catch (Exception e) {
String msg = "Failed to do close consistency check for tablet " + extent;
@@ -1412,12 +1426,10 @@ public class Tablet {
if (!otherLogs.isEmpty() || !currentLogs.isEmpty() ||
!referencedLogs.isEmpty()) {
String msg = "Closed tablet " + extent + " has walog entries in memory
currentLogs = "
- + currentLogs + " otherLogs = " + otherLogs + " refererncedLogs = "
+ referencedLogs;
+ + currentLogs + " otherLogs = " + otherLogs + " referencedLogs = "
+ referencedLogs;
log.error(msg);
throw new RuntimeException(msg);
}
-
-// TODO check lastFlushID and lostCompactID - ACCUMULO-1290
}
private void compareToDataInMemory(TabletMetadata tabletMetadata) {
[accumulo-examples] branch main updated: some minor code cleanup (#83)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 6c00d0c some minor code cleanup (#83)
6c00d0c is described below
commit 6c00d0cc9e717190205ea4252788bca9224dd75a
Author: Jeffrey Manno
AuthorDate: Thu Dec 9 09:06:04 2021 -0500
some minor code cleanup (#83)
---
.../examples/client/CountingVerifyingReceiver.java | 2 +-
.../apache/accumulo/examples/client/RandomBatchWriter.java | 6 +++---
.../apache/accumulo/examples/client/TracingExample.java| 2 +-
.../org/apache/accumulo/examples/dirlist/FileCount.java| 14 +++---
.../java/org/apache/accumulo/examples/dirlist/Ingest.java | 2 +-
.../org/apache/accumulo/examples/dirlist/QueryUtil.java| 12 ++--
.../java/org/apache/accumulo/examples/dirlist/Viewer.java | 4 ++--
.../accumulo/examples/filedata/CharacterHistogram.java | 3 +--
.../accumulo/examples/filedata/ChunkInputFormat.java | 2 +-
.../accumulo/examples/filedata/ChunkInputStream.java | 9 +++--
.../apache/accumulo/examples/filedata/FileDataQuery.java | 2 +-
.../accumulo/examples/filedata/VisibilityCombiner.java | 2 +-
.../apache/accumulo/examples/mapreduce/NGramIngest.java| 8
.../apache/accumulo/examples/mapreduce/TeraSortIngest.java | 14 +++---
.../apache/accumulo/examples/mapreduce/UniqueColumns.java | 2 +-
.../apache/accumulo/examples/shard/ContinuousQuery.java| 2 +-
.../java/org/apache/accumulo/examples/shard/Index.java | 2 +-
.../java/org/apache/accumulo/examples/shard/Query.java | 6 +++---
.../examples/constraints/AlphaNumKeyConstraintTest.java| 2 +-
.../examples/constraints/NumericValueConstraintTest.java | 2 +-
.../accumulo/examples/filedata/ChunkCombinerTest.java | 2 +-
.../accumulo/examples/filedata/ChunkInputFormatIT.java | 6 ++
22 files changed, 50 insertions(+), 56 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java
b/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java
index 9b3d909..1744795 100644
---
a/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java
+++
b/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java
@@ -46,7 +46,7 @@ class CountingVerifyingReceiver {
String row = key.getRow().toString();
long rowid = Integer.parseInt(row.split("_")[1]);
-byte expectedValue[] = RandomBatchWriter.createValue(rowid,
expectedValueSize);
+byte[] expectedValue = RandomBatchWriter.createValue(rowid,
expectedValueSize);
if (!Arrays.equals(expectedValue, value.get())) {
log.error("Got unexpected value for " + key + " expected : "
diff --git
a/src/main/java/org/apache/accumulo/examples/client/RandomBatchWriter.java
b/src/main/java/org/apache/accumulo/examples/client/RandomBatchWriter.java
index a004aec..03b6fbb 100644
--- a/src/main/java/org/apache/accumulo/examples/client/RandomBatchWriter.java
+++ b/src/main/java/org/apache/accumulo/examples/client/RandomBatchWriter.java
@@ -58,7 +58,7 @@ public class RandomBatchWriter {
*/
public static byte[] createValue(long rowid, int dataSize) {
Random r = new Random(rowid);
-byte value[] = new byte[dataSize];
+byte[] value = new byte[dataSize];
r.nextBytes(value);
@@ -89,7 +89,7 @@ public class RandomBatchWriter {
// create a random value that is a function of the
// row id for verification purposes
-byte value[] = createValue(rowid, dataSize);
+byte[] value = createValue(rowid, dataSize);
m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
@@ -159,7 +159,7 @@ public class RandomBatchWriter {
if (e.getSecurityErrorCodes().size() > 0) {
HashMap> tables = new HashMap<>();
for (Entry> ke :
e.getSecurityErrorCodes().entrySet()) {
- String tableId = ke.getKey().getTableId().toString();
+ String tableId = ke.getKey().getTable().toString();
Set secCodes = tables.get(tableId);
if (secCodes == null) {
secCodes = new HashSet<>();
diff --git
a/src/main/java/org/apache/accumulo/examples/client/TracingExample.java
b/src/main/java/org/apache/accumulo/examples/client/TracingExample.java
index 551b43c..5891b96 100644
--- a/src/main/java/org/apache/accumulo/examples/client/TracingExample.java
+++ b/src/main/java/org/apache/accumulo/examples/client/TracingExample.java
@@ -52,7 +52,7 @@ public class TracingExample {
private static final Logger log =
LoggerFactory.getLogger(TracingExample.class);
private static final String DEFAULT_TABLE_NAME = "test";
- private AccumuloClient client;
+ private
[accumulo-testing] branch main updated: Some minor code cleanup throughout accumulo-testing (#176)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 0a50e1f Some minor code cleanup throughout accumulo-testing (#176)
0a50e1f is described below
commit 0a50e1f4d93c135c9cbae90b20628bd48c52e583
Author: Jeffrey Manno
AuthorDate: Tue Dec 7 10:02:12 2021 -0500
Some minor code cleanup throughout accumulo-testing (#176)
---
bin/mapred | 3 +-
.../java/org/apache/accumulo/testing/TestEnv.java | 2 +-
.../testing/continuous/UndefinedAnalyzer.java | 4 +-
.../org/apache/accumulo/testing/gcs/Collector.java | 2 +-
.../org/apache/accumulo/testing/gcs/Generator.java | 2 +-
.../org/apache/accumulo/testing/gcs/Mutator.java | 2 +-
.../apache/accumulo/testing/gcs/Persistence.java | 6 +-
.../org/apache/accumulo/testing/gcs/Verifier.java | 2 +-
.../accumulo/testing/healthprobe/Monitor.java | 11 ++--
.../apache/accumulo/testing/ingest/TestIngest.java | 17 ++
.../accumulo/testing/ingest/VerifyIngest.java | 2 +-
.../testing/performance/tests/GroupCommitPT.java | 14 ++---
.../performance/tests/YieldingScanExecutorPT.java | 5 +-
.../testing/performance/util/TestExecutor.java | 4 +-
.../accumulo/testing/randomwalk/Framework.java | 2 +-
.../apache/accumulo/testing/randomwalk/Module.java | 69 ++
.../apache/accumulo/testing/randomwalk/State.java | 10 ++--
.../accumulo/testing/randomwalk/bulk/Verify.java | 2 +-
.../testing/randomwalk/concurrent/BatchWrite.java | 5 +-
.../testing/randomwalk/concurrent/BulkImport.java | 7 +--
.../testing/randomwalk/concurrent/CreateTable.java | 2 +-
.../testing/randomwalk/conditional/Verify.java | 3 +-
.../accumulo/testing/randomwalk/image/Commit.java | 2 +-
.../testing/randomwalk/image/ImageFixture.java | 6 +-
.../accumulo/testing/randomwalk/image/TableOp.java | 2 +-
.../accumulo/testing/randomwalk/image/Verify.java | 2 +-
.../testing/randomwalk/multitable/Commit.java | 2 +-
.../randomwalk/multitable/MultiTableFixture.java | 6 +-
.../randomwalk/security/WalkingSecurity.java | 11 ++--
.../testing/randomwalk/sequential/BatchVerify.java | 9 +--
.../testing/randomwalk/sequential/Commit.java | 2 +-
.../randomwalk/sequential/MapRedVerifyTool.java| 2 +-
.../randomwalk/sequential/SequentialFixture.java | 4 +-
.../testing/randomwalk/shard/CloneIndex.java | 2 +-
.../accumulo/testing/randomwalk/shard/Grep.java| 9 ++-
.../testing/randomwalk/shard/SortTool.java | 8 +--
.../org/apache/accumulo/testing/stress/Scan.java | 4 +-
37 files changed, 106 insertions(+), 141 deletions(-)
diff --git a/bin/mapred b/bin/mapred
index 269c729..33cb8d2 100755
--- a/bin/mapred
+++ b/bin/mapred
@@ -16,7 +16,6 @@
# limitations under the License.
bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-at_home=$( cd "$( dirname "$bin_dir" )" && pwd )
source "${bin_dir}/build"
function print_usage() {
@@ -55,7 +54,7 @@ export
CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
if [ -n "$HADOOP_HOME" ]; then
export HADOOP_USE_CLIENT_CLASSLOADER=true
- "$HADOOP_HOME"/bin/yarn jar "$TEST_JAR_PATH" "$mr_main" "$TEST_PROPS"
"$ACCUMULO_CLIENT_PROPS" "${@:2}"
+ "$HADOOP_HOME"/bin/yarn jar "$TEST_JAR_PATH" "$mr_main" "$TEST_PROPS"
"$ACCUMULO_CLIENT_PROPS" "${@:2}"
else
echo "Hadoop must be installed and HADOOP_HOME must be set!"
exit 1
diff --git a/src/main/java/org/apache/accumulo/testing/TestEnv.java
b/src/main/java/org/apache/accumulo/testing/TestEnv.java
index d395ad4..18f6afc 100644
--- a/src/main/java/org/apache/accumulo/testing/TestEnv.java
+++ b/src/main/java/org/apache/accumulo/testing/TestEnv.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.conf.Configuration;
public class TestEnv implements AutoCloseable {
protected final Properties testProps;
- private String clientPropsPath;
+ private final String clientPropsPath;
private final Properties clientProps;
private AccumuloClient client = null;
private Configuration hadoopConfig = null;
diff --git
a/src/main/java/org/apache/accumulo/testing/continuous/UndefinedAnalyzer.java
b/src/main/java/org/apache/accumulo/testing/continuous/UndefinedAnalyzer.java
index 61d28ea..00bbe56 100644
---
a/src/main/java/org/apache/accumulo/testing/continuous/UndefinedAnalyzer.java
+++
b/src/main/java/org/apache/accumulo/testing/continuous/UndefinedAnalyzer.java
@@ -319,8 +319,8 @@ public class UndefinedAnalyzer {
if (ta == null)
logger.debug("{} {} {} {
[accumulo-testing] branch main updated: Update zookeeper version to fix Overlapping warning (#174)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 638db89 Update zookeeper version to fix Overlapping warning (#174)
638db89 is described below
commit 638db89039f9a2b7679681a08c848c05b761fc1f
Author: Jeffrey Manno
AuthorDate: Tue Nov 30 11:49:53 2021 -0500
Update zookeeper version to fix Overlapping warning (#174)
---
conf/env.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/env.sh b/conf/env.sh
index aab56dc..d1c85d4 100644
--- a/conf/env.sh
+++ b/conf/env.sh
@@ -61,7 +61,7 @@ fi
# Versions set below will be what is included in the shaded jar
ACCUMULO_VERSION="$("$ACCUMULO_HOME"/bin/accumulo version | grep -v 'DEBUG')";
export ACCUMULO_VERSION
HADOOP_VERSION="$(hadoop version | head -n1 | awk '{print $2}')"; export
HADOOP_VERSION
-export ZOOKEEPER_VERSION=3.4.14
+export ZOOKEEPER_VERSION=3.5.9
# Path to shaded test jar
at_home=$( cd "$( dirname "$conf_dir" )" && pwd )
export TEST_JAR_PATH="${at_home}/target/accumulo-testing-shaded.jar"
[accumulo-testing] branch main updated (bbab863 -> 51bb9bc)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from bbab863 Fix module-info warnings add dc18e21 Fix build warnings in a few classes due to deprication add 3d389cf one more add e09a339 Update src/main/java/org/apache/accumulo/testing/performance/impl/PerfTestRunner.java add cb058df format new 51bb9bc Fix build warnings in a few classes due to depreciation The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/testing/performance/impl/MergeSiteConfig.java | 2 +- .../org/apache/accumulo/testing/performance/impl/PerfTestRunner.java | 2 +- src/main/java/org/apache/accumulo/testing/randomwalk/Framework.java| 2 +- src/main/java/org/apache/accumulo/testing/randomwalk/Module.java | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-)
[accumulo-testing] 01/01: Fix build warnings in a few classes due to depreciation
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit 51bb9bc33a5e75fda787ecb96e7eef47c9469844 Merge: bbab863 cb058df Author: Jeffrey Manno AuthorDate: Wed Nov 24 17:33:00 2021 -0500 Fix build warnings in a few classes due to depreciation .../org/apache/accumulo/testing/performance/impl/MergeSiteConfig.java | 2 +- .../org/apache/accumulo/testing/performance/impl/PerfTestRunner.java | 2 +- src/main/java/org/apache/accumulo/testing/randomwalk/Framework.java| 2 +- src/main/java/org/apache/accumulo/testing/randomwalk/Module.java | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-)
[accumulo-testing] branch main updated (ccce2d6 -> bbab863)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from ccce2d6 Use correct createFixedThreadPool function add 7b909fb Fix zookeeper overlapping classes and module-info warnings add 3d02b8e remove overlapping fix due to bug with perf test new bbab863 Fix module-info warnings The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 1 + 1 file changed, 1 insertion(+)
[accumulo-testing] 01/01: Fix module-info warnings
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit bbab8632af36cf9261f67de6ce1851940412391b Merge: ccce2d6 3d02b8e Author: Jeffrey Manno AuthorDate: Wed Nov 24 17:31:34 2021 -0500 Fix module-info warnings pom.xml | 1 + 1 file changed, 1 insertion(+)
[accumulo-testing] branch main updated (711d28e -> ccce2d6)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from 711d28e Add deletes to continuous ingest (#166) add 33448f3 Use correct createFixedThreadPool function new ccce2d6 Use correct createFixedThreadPool function The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/main/java/org/apache/accumulo/testing/TestProps.java | 3 ++- src/main/java/org/apache/accumulo/testing/randomwalk/Module.java | 2 +- src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-)
[accumulo-testing] 01/01: Use correct createFixedThreadPool function
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit ccce2d672c23445f9befd70acab09252f8ec5825 Merge: 711d28e 33448f3 Author: Jeffrey Manno AuthorDate: Tue Nov 23 15:06:36 2021 -0500 Use correct createFixedThreadPool function src/main/java/org/apache/accumulo/testing/TestProps.java | 3 ++- src/main/java/org/apache/accumulo/testing/randomwalk/Module.java | 2 +- src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Setup.java | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-)
[accumulo] branch main updated: Handle case where manager isn't active due to upgrade (#2321)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new bbc0021 Handle case where manager isn't active due to upgrade (#2321)
bbc0021 is described below
commit bbc002116465be98acc09d30f9e4d1b9486fe944
Author: Jeffrey Manno
AuthorDate: Thu Oct 28 09:47:36 2021 -0400
Handle case where manager isn't active due to upgrade (#2321)
* Add boolean to keep track of UpgradeStatus inside Manager
* Create a isUpgrading function inside HighlyAvailableService to return
that booleans value
* Revise Thrift exception to include a description of the error.
* Throw this exception if trying to access manager while it is upgrading.
---
.../thrift/ThriftNotActiveServiceException.java| 220 -
core/src/main/thrift/client.thrift | 5 +-
.../accumulo/server/HighlyAvailableService.java| 8 +
.../HighlyAvailableServiceInvocationHandler.java | 11 +-
.../java/org/apache/accumulo/manager/Manager.java | 14 ++
.../org/apache/accumulo/tserver/TabletServer.java | 1 -
6 files changed, 255 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/clientImpl/thrift/ThriftNotActiveServiceException.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/clientImpl/thrift/ThriftNotActiveServiceException.java
index 0d86beb..845b8ca 100644
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/clientImpl/thrift/ThriftNotActiveServiceException.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/clientImpl/thrift/ThriftNotActiveServiceException.java
@@ -28,14 +28,19 @@ package org.apache.accumulo.core.clientImpl.thrift;
public class ThriftNotActiveServiceException extends
org.apache.thrift.TException implements
org.apache.thrift.TBase, java.io.Serializable, Cloneable,
Comparable {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("ThriftNotActiveServiceException");
+ private static final org.apache.thrift.protocol.TField SERV_FIELD_DESC = new
org.apache.thrift.protocol.TField("serv",
org.apache.thrift.protocol.TType.STRING, (short)1);
+ private static final org.apache.thrift.protocol.TField
DESCRIPTION_FIELD_DESC = new org.apache.thrift.protocol.TField("description",
org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.scheme.SchemeFactory
STANDARD_SCHEME_FACTORY = new
ThriftNotActiveServiceExceptionStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory
TUPLE_SCHEME_FACTORY = new ThriftNotActiveServiceExceptionTupleSchemeFactory();
+ public @org.apache.thrift.annotation.Nullable java.lang.String serv; //
required
+ public @org.apache.thrift.annotation.Nullable java.lang.String description;
// required
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
+SERV((short)1, "serv"),
+DESCRIPTION((short)2, "description");
private static final java.util.Map byName = new
java.util.HashMap();
@@ -51,6 +56,10 @@ public class ThriftNotActiveServiceException extends
org.apache.thrift.TExceptio
@org.apache.thrift.annotation.Nullable
public static _Fields findByThriftId(int fieldId) {
switch(fieldId) {
+case 1: // SERV
+ return SERV;
+case 2: // DESCRIPTION
+ return DESCRIPTION;
default:
return null;
}
@@ -90,9 +99,15 @@ public class ThriftNotActiveServiceException extends
org.apache.thrift.TExceptio
return _fieldName;
}
}
+
+ // isset id assignments
public static final java.util.Map<_Fields,
org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap =
new java.util.EnumMap<_Fields,
org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+tmpMap.put(_Fields.SERV, new
org.apache.thrift.meta_data.FieldMetaData("serv",
org.apache.thrift.TFieldRequirementType.DEFAULT,
+new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+tmpMap.put(_Fields.DESCRIPTION, new
org.apache.thrift.meta_data.FieldMetaData("description",
org.apache.thrift.TFieldRequirementType.DEFAULT,
+new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ThriftNotActiveServiceException.class,
metaDataMap);
}
@@ -100,10 +115
[accumulo-testing] branch main updated (c2c2408 -> fb59862)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from c2c2408 Fix failing ScanExecutorPT (#163) add 732047c ignore replication IT inside accumulo-testing new fb59862 Ignore ReplicaitonRandomWalkIT from maven verify build The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/testing/randomwalk/ReplicationRandomWalkIT.java | 2 ++ 1 file changed, 2 insertions(+)
[accumulo-testing] 01/01: Ignore ReplicaitonRandomWalkIT from maven verify build
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit fb59862b3f1f1c5650bf7b17d9336b23ae93588d Merge: c2c2408 732047c Author: Jeffrey Manno AuthorDate: Tue Oct 19 17:40:30 2021 -0400 Ignore ReplicaitonRandomWalkIT from maven verify build .../org/apache/accumulo/testing/randomwalk/ReplicationRandomWalkIT.java | 2 ++ 1 file changed, 2 insertions(+)
[accumulo-testing] branch main updated (86536bb -> 27476d0)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from 86536bb Add units to performance test results (#156) add 5561b7c Add documentation, comments, and warning fixes new 27476d0 Update readme and add comments to RandomCachedLookupsPT The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: README.md| 3 +++ .../testing/performance/tests/RandomCachedLookupsPT.java | 9 ++--- 2 files changed, 9 insertions(+), 3 deletions(-)
[accumulo-testing] 01/01: Update readme and add comments to RandomCachedLookupsPT
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit 27476d0fb855ecaae7a04a0343d0e065434448d1 Merge: 86536bb 5561b7c Author: Jeffrey Manno AuthorDate: Wed Sep 29 09:08:20 2021 -0400 Update readme and add comments to RandomCachedLookupsPT * These additions aim to inform the user that some tests may require changes to certain property values in order to run. README.md| 3 +++ .../testing/performance/tests/RandomCachedLookupsPT.java | 9 ++--- 2 files changed, 9 insertions(+), 3 deletions(-)
[accumulo] branch main updated: Added ColFam and ColQual opts to scan command (#2255)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9dd6580 Added ColFam and ColQual opts to scan command (#2255)
9dd6580 is described below
commit 9dd65807008d3d3ed313d7cfd10bfb68c4ccc597
Author: harjitdotsingh <75006+harjitdotsi...@users.noreply.github.com>
AuthorDate: Thu Sep 16 09:39:03 2021 -0400
Added ColFam and ColQual opts to scan command (#2255)
* This will enable users to have scan for column families with characters
such as colons in them.
* Added tests to verify this functionality.
Co-authored-by: Hsingh
Co-authored-by: Jeffrey Manno
---
.../accumulo/shell/commands/ScanCommand.java | 49 +-
.../accumulo/shell/commands/ScanCommandTest.java | 12 ++
.../org/apache/accumulo/test/ShellServerIT.java| 11 +
3 files changed, 70 insertions(+), 2 deletions(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
index 8f5fe5f..d1c0b8d 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
@@ -59,7 +59,7 @@ import org.apache.hadoop.io.Text;
public class ScanCommand extends Command {
private Option scanOptAuths, scanOptRow, scanOptColumns,
disablePaginationOpt, showFewOpt,
- formatterOpt, interpreterOpt, formatterInterpeterOpt, outputFileOpt;
+ formatterOpt, interpreterOpt, formatterInterpeterOpt, outputFileOpt,
scanOptCf, scanOptCq;
protected Option timestampOpt;
protected Option profileOpt;
@@ -114,6 +114,7 @@ public class ScanCommand extends Command {
// handle columns
fetchColumns(cl, scanner, interpeter);
+ fetchColumsWithCFAndCQ(cl, scanner, interpeter);
// set timeout
scanner.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS);
@@ -270,6 +271,16 @@ public class ScanCommand extends Command {
protected void fetchColumns(final CommandLine cl, final ScannerBase scanner,
final ScanInterpreter formatter) throws UnsupportedEncodingException {
+
+if ((cl.hasOption(scanOptCf.getOpt()) || cl.hasOption(scanOptCq.getOpt()))
+&& cl.hasOption(scanOptColumns.getOpt())) {
+
+ String formattedString =
+ String.format("Option -%s is mutually exclusive with options -%s and
-%s.",
+ scanOptColumns.getOpt(), scanOptCf.getOpt(), scanOptCq.getOpt());
+ throw new IllegalArgumentException(formattedString);
+}
+
if (cl.hasOption(scanOptColumns.getOpt())) {
for (String a : cl.getOptionValue(scanOptColumns.getOpt()).split(",")) {
final String[] sa = a.split(":", 2);
@@ -285,6 +296,32 @@ public class ScanCommand extends Command {
}
}
+ private void fetchColumsWithCFAndCQ(CommandLine cl, Scanner scanner,
ScanInterpreter interpeter) {
+String cf = "";
+String cq = "";
+if (cl.hasOption(scanOptCf.getOpt())) {
+ cf = cl.getOptionValue(scanOptCf.getOpt());
+}
+if (cl.hasOption(scanOptCq.getOpt())) {
+ cq = cl.getOptionValue(scanOptCq.getOpt());
+}
+
+if (cf.isEmpty() && !cq.isEmpty()) {
+ String formattedString = String.format("Option -%s is required when
using -%s.",
+ scanOptCf.getOpt(), scanOptCq.getOpt());
+ throw new IllegalArgumentException(formattedString);
+} else if (!cf.isEmpty() && cq.isEmpty()) {
+ scanner.fetchColumnFamily(
+ interpeter.interpretColumnFamily(new
Text(cf.getBytes(Shell.CHARSET;
+
+} else if (!cf.isEmpty() && !cq.isEmpty()) {
+ scanner.fetchColumn(interpeter.interpretColumnFamily(new
Text(cf.getBytes(Shell.CHARSET))),
+ interpeter.interpretColumnQualifier(new
Text(cq.getBytes(Shell.CHARSET;
+
+}
+
+ }
+
protected Range getRange(final CommandLine cl, final ScanInterpreter
formatter)
throws UnsupportedEncodingException {
if ((cl.hasOption(OptUtil.START_ROW_OPT) ||
cl.hasOption(OptUtil.END_ROW_OPT))
@@ -347,7 +384,11 @@ public class ScanCommand extends Command {
"make end row exclusive (by default it's inclusive)");
optEndRowExclusive.setArgName("end-exclusive");
scanOptRow = new Option("r", "row", true, "row to scan");
-scanOptColumns = new Option("c", "columns", true, "comma-separated
columns");
+scanOptColumns = new Option("c", "columns", true,
+"comma-separated columns.This" + " option is mutually exclusive with
cf and cq");
+scanOptCf = new Option("cf", "column-family",
[accumulo] branch main updated: Minor changes to compactable error message template (#2243)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4222d88 Minor changes to compactable error message template (#2243)
4222d88 is described below
commit 4222d8828f4236c2d3834f0948b24f6bee86ba22
Author: Jeffrey Manno
AuthorDate: Tue Aug 24 16:43:29 2021 -0400
Minor changes to compactable error message template (#2243)
Co-authored-by: Dom G. <47725857+domgargu...@users.noreply.github.com>
Co-authored-by: Mike Miller
---
.../main/java/org/apache/accumulo/tserver/compactions/Compactable.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
index be70321..76307dd 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/Compactable.java
@@ -76,7 +76,7 @@ public interface Compactable {
var compactingFiles =
compacting.stream().flatMap(job ->
job.getFiles().stream()).collect(Collectors.toSet());
Preconditions.checkArgument(this.allFiles.containsAll(compactingFiles),
- "Compacting not in set of all files %s %s", this.allFiles,
compactingFiles);
+ "Compacting files %s not in set of all files: %s", compactingFiles,
this.allFiles);
Preconditions.checkArgument(Collections.disjoint(compactingFiles,
this.candidates),
"Compacting and candidates overlap %s %s", compactingFiles,
this.candidates);
[accumulo] branch main updated: Remove use of JLine outside of accumulo-shell (#2228)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7db0561 Remove use of JLine outside of accumulo-shell (#2228)
7db0561 is described below
commit 7db0561cac3d42ba09d1fb038dad1ff1a03afc46
Author: harjitdotsingh <75006+harjitdotsi...@users.noreply.github.com>
AuthorDate: Wed Aug 11 17:32:19 2021 -0400
Remove use of JLine outside of accumulo-shell (#2228)
* Includes general refactors to Initialize.java
---
core/pom.xml | 4 -
.../org/apache/accumulo/core/util/CreateToken.java | 12 ++-
server/base/pom.xml| 4 -
.../apache/accumulo/server/init/Initialize.java| 86 +-
4 files changed, 40 insertions(+), 66 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 4388500..575bdd3 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -126,10 +126,6 @@
zookeeper-jute
- org.jline
- jline
-
-
org.slf4j
slf4j-api
diff --git a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
index c92cdd2..8306f64 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
@@ -18,7 +18,7 @@
*/
package org.apache.accumulo.core.util;
-import java.io.IOException;
+import java.io.Console;
import org.apache.accumulo.core.cli.ClientOpts.PasswordConverter;
import org.apache.accumulo.core.cli.Help;
@@ -28,8 +28,6 @@ import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Token
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.ClientProperty;
import org.apache.accumulo.start.spi.KeywordExecutable;
-import org.jline.reader.LineReader;
-import org.jline.reader.LineReaderBuilder;
import com.beust.jcommander.Parameter;
import com.google.auto.service.AutoService;
@@ -37,12 +35,12 @@ import com.google.auto.service.AutoService;
@AutoService(KeywordExecutable.class)
public class CreateToken implements KeywordExecutable {
- private LineReader reader = null;
+ private Console reader = null;
- private LineReader getConsoleReader() throws IOException {
+ private Console getConsoleReader() {
if (reader == null)
- reader = LineReaderBuilder.builder().build();
+ reader = System.console();
return reader;
}
@@ -113,7 +111,7 @@ public class CreateToken implements KeywordExecutable {
System.out.println("auth.type = " + opts.tokenClassName);
System.out.println("auth.principal = " + principal);
System.out.println("auth.token = " + ClientProperty.encodeToken(token));
-} catch (IOException | ReflectiveOperationException e) {
+} catch (ReflectiveOperationException e) {
throw new RuntimeException(e);
}
}
diff --git a/server/base/pom.xml b/server/base/pom.xml
index fda74eb..eca7744 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -93,10 +93,6 @@
zookeeper-jute
- org.jline
- jline
-
-
org.slf4j
slf4j-api
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 8521a8d..71f03dc 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -119,11 +119,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
-import org.jline.reader.LineReader;
-import org.jline.reader.LineReaderBuilder;
-import org.jline.terminal.Terminal;
-import org.jline.terminal.TerminalBuilder;
-import org.jline.utils.InfoCmp;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,18 +139,8 @@ public class Initialize implements KeywordExecutable {
private static final String DEFAULT_ROOT_USER = "root";
private static final String TABLE_TABLETS_TABLET_DIR = "table_info";
- private static LineReader reader = null;
- private static Terminal terminal = null;
private static ZooReaderWriter zoo = null;
- private static LineReader getLineReader() throws IOException {
-if (terminal == null)
- terminal = TerminalBuilder.builder().jansi(false).build();
-if (reader == null)
- reader = LineReaderBuilder.builder().terminal(terminal).build();
-return reader;
- }
-
/**
* Sets this class's ZooKeeper reader/writer.
*
@@ -279,18 +264,17 @@ public class Initi
[accumulo] branch main updated: Remove unnecessary line in javadoc of TransformingIterator (#2229)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9ae9337 Remove unnecessary line in javadoc of TransformingIterator
(#2229)
9ae9337 is described below
commit 9ae9337fcf0aae732cd05fb88871196996c04242
Author: Jeffrey Manno
AuthorDate: Wed Aug 11 10:22:47 2021 -0400
Remove unnecessary line in javadoc of TransformingIterator (#2229)
---
.../org/apache/accumulo/core/iterators/user/TransformingIterator.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 3cf6487..a6e41f1 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@ -72,8 +72,7 @@ import org.slf4j.LoggerFactory;
* fetch column families that are known to not transform at all).
*
* If the implementing iterator is transforming column visibilities, then
users must be careful NOT
- * to fetch column qualifiers from the scanner. The reason for this is due to
ACCUMULO-??? (insert
- * issue number).
+ * to fetch column qualifiers from the scanner.
*
* If the implementing iterator is transforming column visibilities, then the
user should be sure to
* supply authorizations via the {@link #AUTH_OPT} iterator option (note that
this is only necessary
[accumulo-testing] 01/01: Replace deprecated prop inside ReplicationRandomWalkIT with its replacement
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit e704f40843d7aa68f2443f8599865ecfd6e836fc Merge: a2a1ebb e918397 Author: Jeffrey Manno AuthorDate: Mon Aug 9 12:06:26 2021 -0400 Replace deprecated prop inside ReplicationRandomWalkIT with its replacement .../apache/accumulo/testing/randomwalk/ReplicationRandomWalkIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[accumulo-testing] branch main updated (a2a1ebb -> e704f40)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from a2a1ebb Properly build shaded jar when running cingest (#140) add e918397 Replace deprecated prop inside ReplicationRandomWalkIT new e704f40 Replace deprecated prop inside ReplicationRandomWalkIT with its replacement The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/testing/randomwalk/ReplicationRandomWalkIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[accumulo] branch main updated: Pretty-print PrintInfo's 'histogram' output (#2220)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1e9ac21 Pretty-print PrintInfo's 'histogram' output (#2220)
1e9ac21 is described below
commit 1e9ac217e36813b9f5a1d2c904ee790fbbea8f43
Author: Shivakumar <47275182+shivakumar...@users.noreply.github.com>
AuthorDate: Wed Aug 4 22:51:10 2021 +0530
Pretty-print PrintInfo's 'histogram' output (#2220)
* PrettyPrint printInfo's 'histogram' output to be more human-readable.
Co-authored-by: Jeffrey Manno
---
.../main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 4423a0c..55d188c 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -40,6 +40,7 @@ import org.apache.accumulo.core.file.rfile.bcfile.Utils;
import org.apache.accumulo.core.spi.crypto.NoFileEncrypter;
import org.apache.accumulo.core.summary.SummaryReader;
import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.NumUtil;
import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
@@ -106,10 +107,11 @@ public class PrintInfo implements KeywordExecutable {
}
public void print(String indent) {
- System.out.println(indent + "Up to size count %-age");
+ System.out.println(indent + "Up to size Count %-age");
for (int i = 1; i < countBuckets.length; i++) {
-System.out.println(String.format("%s%11.0f : %10d %6.2f%%", indent,
Math.pow(10, i),
-countBuckets[i], sizeBuckets[i] * 100. / totalSize));
+System.out.println(String.format("%s%11s : %10d %6.2f%%", indent,
+NumUtil.bigNumberForQuantity((long) Math.pow(10, i)),
countBuckets[i],
+sizeBuckets[i] * 100. / totalSize));
}
}
}
[accumulo] branch main updated: Fix initialize.java by using terminalBuilder instead of default (#2202)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new f5df4f9 Fix initialize.java by using terminalBuilder instead of
default (#2202)
f5df4f9 is described below
commit f5df4f9681736d35e14b92c58033a1cf062b3d4e
Author: Jeffrey Manno
AuthorDate: Thu Jul 15 13:25:24 2021 -0400
Fix initialize.java by using terminalBuilder instead of default (#2202)
---
.../main/java/org/apache/accumulo/server/init/Initialize.java | 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index ae6448a..8521a8d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -121,6 +121,8 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.jline.reader.LineReader;
import org.jline.reader.LineReaderBuilder;
+import org.jline.terminal.Terminal;
+import org.jline.terminal.TerminalBuilder;
import org.jline.utils.InfoCmp;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -143,12 +145,14 @@ public class Initialize implements KeywordExecutable {
private static final String TABLE_TABLETS_TABLET_DIR = "table_info";
private static LineReader reader = null;
+ private static Terminal terminal = null;
private static ZooReaderWriter zoo = null;
private static LineReader getLineReader() throws IOException {
-if (reader == null) {
- reader = LineReaderBuilder.builder().build();
-}
+if (terminal == null)
+ terminal = TerminalBuilder.builder().jansi(false).build();
+if (reader == null)
+ reader = LineReaderBuilder.builder().terminal(terminal).build();
return reader;
}
[accumulo] branch main updated: Update LogReader to utilize RecoveryLogsIterator (#2181)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new f9f1d3f Update LogReader to utilize RecoveryLogsIterator (#2181)
f9f1d3f is described below
commit f9f1d3f6578c400862e9f0e8d82a49bf9c4a5392
Author: Jeffrey Manno
AuthorDate: Wed Jul 14 08:15:52 2021 -0400
Update LogReader to utilize RecoveryLogsIterator (#2181)
* Adds utilization of RecoveryLogsIterator to read sorted Rfiles inside
LogReader.java
* Removed old implementation of RecoveryLogReader and removed associated
test
* Added unit test for RecoveryLogsIterator, RecoveryLogsIteratorTest
Co-authored-by: Christopher Tubbs
---
.../accumulo/tserver/log/RecoveryLogReader.java| 326 -
.../accumulo/tserver/log/RecoveryLogsIterator.java | 4 +-
.../apache/accumulo/tserver/logger/LogReader.java | 31 +-
.../tserver/log/RecoveryLogsIteratorTest.java | 251
.../tserver/log/RecoveryLogsReaderTest.java| 216 --
5 files changed, 273 insertions(+), 555 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java
deleted file mode 100644
index bde5a1c..000
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogReader.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.tserver.log;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.AbstractMap;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.PriorityQueue;
-
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.log.SortedLogState;
-import org.apache.accumulo.tserver.logger.LogEvents;
-import org.apache.accumulo.tserver.logger.LogFileKey;
-import org.apache.accumulo.tserver.logger.LogFileValue;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.MapFile.Reader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-
-/**
- * A class which reads sorted recovery logs produced from a single WAL.
- *
- * Presently only supports next() and seek() and works on all the Map
directories within a
- * directory. The primary purpose of this class is to merge the results of
multiple Reduce jobs that
- * result in Map output files.
- */
-public class RecoveryLogReader implements
CloseableIterator> {
-
- /**
- * Group together the next key/value from a Reader with the Reader
- */
- private static class Index implements Comparable {
-Reader reader;
-WritableComparable key;
-Writable value;
-boolean cached = false;
-
-private static Object create(java.lang.Class klass) {
- try {
-return klass.getConstructor().newInstance();
- } catch (Exception t) {
-throw new RuntimeException("Unable to construct objects to use for
comparison");
- }
-}
-
-public Index(Reader reader) {
- this.reader = reader;
- key = (WritableComparable) create(reader.getKeyClass());
- value = (Writable) create(reader.getValueClass());
-}
-
-private void cache() throws IOException {
- if (!cached && reader.next(key, value)) {
-cached = true;
- }
-}
-
-@Override
-public int hashCode() {
- return Objects.hashCode(key);
-}
-
-@Override
-public boolean equals(Object obj) {
- return this == obj || (obj != null && obj
[accumulo] branch main updated (908105b -> 7bbe579)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 908105b Refactor LogSorter to use config (#2191) add 7bbe579 Update CheckForMetadataProblems Utility Output (#2180) No new revisions were added by this update. Summary of changes: .../server/util/CheckForMetadataProblems.java | 48 ++ 1 file changed, 31 insertions(+), 17 deletions(-)
[accumulo] branch main updated: Add retry counter for log recovery with MinC (#2084)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9d42e3b Add retry counter for log recovery with MinC (#2084)
9d42e3b is described below
commit 9d42e3bf4cb3650b72b680f91855eeda6ff6ee53
Author: Jeffrey Manno
AuthorDate: Wed Jun 16 13:46:41 2021 -0400
Add retry counter for log recovery with MinC (#2084)
* Add retrycounter for log recovery with MinC
---
.../apache/accumulo/tserver/tablet/MinorCompactor.java | 18 +++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
index 37dae21..c49f6d5 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
@@ -62,6 +62,7 @@ public class MinorCompactor extends FileCompactor {
private static final Logger log =
LoggerFactory.getLogger(MinorCompactor.class);
private final TabletServer tabletServer;
+ private final MinorCompactionReason mincReason;
public MinorCompactor(TabletServer tabletServer, Tablet tablet, InMemoryMap
imm,
TabletFile outputFile, MinorCompactionReason mincReason,
TableConfiguration tableConfig) {
@@ -113,6 +114,7 @@ public class MinorCompactor extends FileCompactor {
}
}, Collections.emptyList(), tableConfig);
this.tabletServer = tabletServer;
+this.mincReason = mincReason;
}
private boolean isTableDeleting() {
@@ -141,6 +143,7 @@ public class MinorCompactor extends FileCompactor {
double growthFactor = 4;
int maxSleepTime = 1000 * 60 * 3; // 3 minutes
boolean reportedProblem = false;
+int retryCounter = 0;
runningCompactions.add(this);
try {
@@ -165,12 +168,21 @@ public class MinorCompactor extends FileCompactor {
reportedProblem = true;
} catch (RuntimeException | NoClassDefFoundError e) {
// if this is coming from a user iterator, it is possible that the
user could change the
- // iterator config and that the
- // minor compaction would succeed
- log.warn("MinC failed ({}) to create {} retrying ...",
e.getMessage(), outputFileName, e);
+ // iterator config and that the minor compaction would succeed
+ // If the minor compaction stalls for too long during recovery, it
can interfere with
+ // other tables loading
+ // Throw exception if this happens so assignments can be rescheduled.
ProblemReports.getInstance(tabletServer.getContext()).report(
new ProblemReport(getExtent().tableId(), ProblemType.FILE_WRITE,
outputFileName, e));
+ if (retryCounter >= 4 &&
mincReason.equals(MinorCompactionReason.RECOVERY)) {
+log.warn(
+"MinC ({}) is stuck for too long during recovery, throwing
error to reschedule.",
+getExtent(), e);
+throw new RuntimeException(e);
+ }
+ log.warn("MinC failed ({}) to create {} retrying ...",
e.getMessage(), outputFileName, e);
reportedProblem = true;
+ retryCounter++;
} catch (CompactionCanceledException e) {
throw new IllegalStateException(e);
}
[accumulo-website] branch main updated: Update tools doc for CheckForMetadataProblems (#285)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new c386fc7 Update tools doc for CheckForMetadataProblems (#285) c386fc7 is described below commit c386fc74b84831a71d4bcbae3a11dece13b2bc7d Author: Jeffrey Manno AuthorDate: Tue Jun 15 10:47:45 2021 -0400 Update tools doc for CheckForMetadataProblems (#285) --- _docs-2/troubleshooting/tools.md | 5 + 1 file changed, 5 insertions(+) diff --git a/_docs-2/troubleshooting/tools.md b/_docs-2/troubleshooting/tools.md index 817524d..7083c21 100644 --- a/_docs-2/troubleshooting/tools.md +++ b/_docs-2/troubleshooting/tools.md @@ -102,8 +102,13 @@ every tablet matches, and the start and stop for the table is empty: $ accumulo org.apache.accumulo.server.util.CheckForMetadataProblems -u root --password Enter the connection password: +Checking table: accumulo.root All is well for table !0 +No problems found +Checking table: accumulo.metadata +All is well for table +rep All is well for table 1 +No problems found ## RemoveEntriesForMissingFiles
[accumulo] branch main updated (987fc0c -> d3b912c)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 987fc0c Specify blockName in CachedBlock exceptions (#2163) add d3b912c Refactor code to take advantage of Value no-arg constructor (#2162) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/core/iterators/SortedKeyIterator.java | 2 +- .../core/constraints/DefaultKeySizeConstraintTest.java | 10 +- .../core/constraints/DeprecatedConstraintExtendTest.java | 10 +- .../test/java/org/apache/accumulo/core/crypto/CryptoTest.java | 4 ++-- .../core/data/constraints/DefaultKeySizeConstraintTest.java| 10 +- .../org/apache/accumulo/core/file/rfile/BlockIndexTest.java| 8 .../accumulo/core/iterators/system/VisibilityFilterTest.java | 8 .../accumulo/core/iterators/user/IndexedDocIteratorTest.java | 2 +- .../accumulo/core/iterators/user/IntersectingIteratorTest.java | 2 +- .../apache/accumulo/core/iterators/user/RegExFilterTest.java | 2 +- .../accumulo/core/iterators/user/VisibilityFilterTest.java | 2 +- .../org/apache/accumulo/server/metadata/TabletMutatorBase.java | 2 +- .../main/java/org/apache/accumulo/server/util/FileUtil.java| 2 +- .../org/apache/accumulo/server/client/BulkImporterTest.java| 2 +- .../apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java | 2 +- .../tserver/replication/AccumuloReplicaSystemTest.java | 4 ++-- .../java/org/apache/accumulo/test/BulkImportMonitoringIT.java | 2 +- .../java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java | 2 +- test/src/main/java/org/apache/accumulo/test/TestIngest.java| 2 +- .../main/java/org/apache/accumulo/test/VolumeChooserIT.java| 2 +- .../accumulo/test/functional/DeletedTablesDontFlushIT.java | 2 +- 21 files changed, 41 insertions(+), 41 deletions(-)
[accumulo] branch main updated (0fc2686 -> d15f21e)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 0fc2686 Fix shellcheck warnings in accumulo-cluster script (#2133) add d15f21e PopulateMetadataTable Improvements (#2118) No new revisions were added by this update. Summary of changes: .../tableImport/PopulateMetadataTable.java | 41 ++ 1 file changed, 11 insertions(+), 30 deletions(-)
[accumulo] branch main updated (c11ee7b -> 196fda5)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from c11ee7b Unit test converting client props to BatchWriterConfig (#2126) add 196fda5 Add Compactor and Coordinator to testExpectedClasses in KeywordStartIT (#2139) No new revisions were added by this update. Summary of changes: test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java | 4 1 file changed, 4 insertions(+)
[accumulo] branch main updated: Fix history test inside ShellServerIT (#2097)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 92c812c Fix history test inside ShellServerIT (#2097)
92c812c is described below
commit 92c812cc5655a0ff0a9c7fa7e578de2586d58d5c
Author: Jeffrey Manno
AuthorDate: Thu May 13 06:23:33 2021 -0400
Fix history test inside ShellServerIT (#2097)
* thoroughly check all features of the 'history' command works.
* create a direct write-to-history function. Previously, things were
not being written to the history file due to how the execution works
for this IT
---
.../org/apache/accumulo/test/ShellServerIT.java| 30 --
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index 081e984..66d8784 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -285,6 +285,11 @@ public class ShellServerIT extends SharedMiniClusterBase {
output.get().contains(s));
shell.resetExitCode();
}
+
+void writeToHistory(String cmd) {
+ input.set(cmd);
+ reader.readLine();
+}
}
private static final NoOpErrorMessageCallback noop = new
NoOpErrorMessageCallback();
@@ -1431,18 +1436,27 @@ public class ShellServerIT extends
SharedMiniClusterBase {
}
}
- // TODO - evaluate this test is testing what is expected or history is
working.
@Test
public void history() throws Exception {
-final String table = name.getMethodName();
+final String table = getUniqueNames(1)[0];
+// test clear history command works
+ts.writeToHistory("foo");
+ts.exec("history", true, "foo", true);
ts.exec("history -c", true);
-ts.exec("createtable " + table);
-ts.exec("deletetable -f " + table);
-// TODO - this may be passing only because method name is history.
-ts.exec("history", true, table, true);
-// TODO - what is this testing?
-ts.exec("history", true, "history", true);
+ts.exec("history", true, "foo", false);
+
+// Verify commands are not currently in history then write to history
file. Does not execute
+// table ops.
+ts.exec("history", true, table, false);
+ts.exec("history", true, "createtable", false);
+ts.exec("history", true, "deletetable", false);
+ts.writeToHistory("createtable " + table);
+ts.writeToHistory("deletetable -f " + table);
+
+// test that history command prints contents of history file
+ts.exec("history", true, "createtable " + table, true);
+ts.exec("history", true, "deletetable -f " + table, true);
}
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path
provided by test")
[accumulo] branch main updated: Additional unit test to exclude from using reuseForks (#2044)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 552fd33 Additional unit test to exclude from using reuseForks (#2044) 552fd33 is described below commit 552fd33407f9346fba45525f3abb36864de2ea09 Author: Jeffrey Manno AuthorDate: Tue May 4 06:11:43 2021 -0400 Additional unit test to exclude from using reuseForks (#2044) * exclude AuthenticationTokenSecretManagerTest from reuseForks=true --- server/base/pom.xml | 27 +++ 1 file changed, 27 insertions(+) diff --git a/server/base/pom.xml b/server/base/pom.xml index d499017..fda74eb 100644 --- a/server/base/pom.xml +++ b/server/base/pom.xml @@ -128,6 +128,33 @@ src/test/resources + + +org.apache.maven.plugins +maven-surefire-plugin + + + +**/AuthenticationTokenSecretManagerTest.java + + + + +reuseForks-false-tests + + test + + + false + + +**/AuthenticationTokenSecretManagerTest.java + + + + + +
[accumulo] branch main updated: Pom changes to allow reuseForks to be set to true (#2015)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 5c4e196 Pom changes to allow reuseForks to be set to true (#2015) 5c4e196 is described below commit 5c4e1962097584e4b4269def73bce6be54c2493b Author: Jeffrey Manno AuthorDate: Wed Apr 21 13:56:01 2021 -0400 Pom changes to allow reuseForks to be set to true (#2015) * Created second execution to run the few tests that cannot run with reuseForks being true. --- core/pom.xml | 33 + pom.xml| 2 +- server/tserver/pom.xml | 25 + 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/core/pom.xml b/core/pom.xml index c5e9552..36d9dc6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -195,6 +195,39 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/TableIdTest.java + **/HadoopCredentialProviderTest.java + **/IdleRatioScanPrioritizerTest.java + **/TestCfCqSlice.java + **/TestCfCqSliceSeekingFilter.java + + + + + reuseForks-false-tests + +test + + +false + + + **/TableIdTest.java + **/HadoopCredentialProviderTest.java + **/IdleRatioScanPrioritizerTest.java + **/TestCfCqSlice.java + **/TestCfCqSliceSeekingFilter.java + + + + + diff --git a/pom.xml b/pom.xml index 54dca69..b41ac03 100644 --- a/pom.xml +++ b/pom.xml @@ -149,7 +149,7 @@ false 1C -false +true 3.0.0-M4 diff --git a/server/tserver/pom.xml b/server/tserver/pom.xml index 362149a..69bb4b0 100644 --- a/server/tserver/pom.xml +++ b/server/tserver/pom.xml @@ -151,6 +151,31 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/AssignmentWatcherTest.java + + + + + reuseForks-false-tests + +test + + +false + + + **/AssignmentWatcherTest.java + + + + +
[accumulo] branch main updated: Set reuseForks to true for unit tests, false for ITs (#2007)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7957d67 Set reuseForks to true for unit tests, false for ITs (#2007)
7957d67 is described below
commit 7957d67b0a0d5571a9174660e6353c9cb421e188
Author: Jeffrey Manno
AuthorDate: Fri Apr 9 12:32:11 2021 -0400
Set reuseForks to true for unit tests, false for ITs (#2007)
* Create surefire and failsafe reuseForks
* Create a profile to be able to set both values from either false or true
---
pom.xml | 36 ++--
1 file changed, 34 insertions(+), 2 deletions(-)
diff --git a/pom.xml b/pom.xml
index 466f720..8501f39 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,7 @@
1
+false
3.3.0
4.1.0-incubating
3.2.0-incubating
@@ -142,14 +143,13 @@
2020-12-17T22:06:50Z
true
-
-false
1.7.30
source-release-tar
false
1C
+true
3.0.0-M4
@@ -814,7 +814,17 @@
maven-surefire-plugin
${surefire.forkCount}
+${surefire.reuseForks}
${surefire.excludedGroups}
+
+
+ **/TableIdTest.java
+ **/HadoopCredentialProviderTest.java
+ **/IdleRatioScanPrioritizerTest.java
+ **/AssignmentWatcherTest.java
+ **/TestCfCqSlice.java
+ **/TestCfCqSliceSeekingFilter.java
+
${surefire.groups}
${project.build.directory}
@@ -827,8 +837,18 @@
maven-failsafe-plugin
${failsafe.forkCount}
+${failsafe.reuseForks}
${failsafe.excludedGroups}
${failsafe.groups}
+
+
+ **/TableIdTest.java
+ **/HadoopCredentialProviderTest.java
+ **/IdleRatioScanPrioritizerTest.java
+ **/AssignmentWatcherTest.java
+ **/TestCfCqSlice.java
+ **/TestCfCqSliceSeekingFilter.java
+
${project.build.directory}
@@ -1554,5 +1574,17 @@
${forkCount}
+
+ reuseForks
+
+
+ reuseForks
+
+
+
+${reuseForks}
+${reuseForks}
+
+
[accumulo] branch main updated: Add created column to Fate Print command (#1980)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new aab34a9 Add created column to Fate Print command (#1980)
aab34a9 is described below
commit aab34a980d7a9ccad63797913b99b331c5cbbaae
Author: Jeffrey Manno
AuthorDate: Wed Mar 31 13:36:10 2021 -0400
Add created column to Fate Print command (#1980)
* Add timestamp of created time based on transaction Id's of FaTE operations
* Create a helper function to format the returned long value of the
timestamp
* Formatted created time within the `fate print` shell command
---
.../apache/accumulo/core/logging/FateLogger.java | 5
.../java/org/apache/accumulo/fate/AdminUtil.java | 31 +++---
.../java/org/apache/accumulo/fate/AgeOffStore.java | 5
.../org/apache/accumulo/fate/ReadOnlyStore.java| 5
.../org/apache/accumulo/fate/ReadOnlyTStore.java | 8 ++
.../java/org/apache/accumulo/fate/ZooStore.java| 13 +
.../java/org/apache/accumulo/fate/SimpleStore.java | 5
7 files changed, 68 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
index c856275..6121bce 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
@@ -86,6 +86,11 @@ public class FateLogger {
}
@Override
+ public long timeCreated(long tid) {
+return store.timeCreated(tid);
+ }
+
+ @Override
public long create() {
long tid = store.create();
if (storeLog.isTraceEnabled())
diff --git a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 3016d01..d4e45a8 100644
--- a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@ -20,8 +20,11 @@ package org.apache.accumulo.fate;
import static java.nio.charset.StandardCharsets.UTF_8;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.EnumSet;
import java.util.Formatter;
import java.util.HashMap;
@@ -80,9 +83,10 @@ public class AdminUtil {
private final List hlocks;
private final List wlocks;
private final String top;
+private final long timeCreated;
private TransactionStatus(Long tid, TStatus status, String debug,
List hlocks,
-List wlocks, String top) {
+List wlocks, String top, Long timeCreated) {
this.txid = tid;
this.status = status;
@@ -90,6 +94,7 @@ public class AdminUtil {
this.hlocks = Collections.unmodifiableList(hlocks);
this.wlocks = Collections.unmodifiableList(wlocks);
this.top = top;
+ this.timeCreated = timeCreated;
}
@@ -132,6 +137,21 @@ public class AdminUtil {
public String getTop() {
return top;
}
+
+/**
+ * @return The timestamp of when the operation was created in ISO format
wiht UTC timezone.
+ */
+public String getTimeCreatedFormatted() {
+ return timeCreated > 0 ? new
Date(timeCreated).toInstant().atZone(ZoneOffset.UTC)
+ .format(DateTimeFormatter.ISO_DATE_TIME) : "ERROR";
+}
+
+/**
+ * @return The unformatted form of the timestamp.
+ */
+public long getTimeCreated() {
+ return timeCreated;
+}
}
public static class FateStatus {
@@ -373,13 +393,15 @@ public class AdminUtil {
TStatus status = zs.getStatus(tid);
+ long timeCreated = zs.timeCreated(tid);
+
zs.unreserve(tid, 0);
if ((filterTxid != null && !filterTxid.contains(tid))
|| (filterStatus != null && !filterStatus.contains(status)))
continue;
- statuses.add(new TransactionStatus(tid, status, debug, hlocks, wlocks,
top));
+ statuses.add(new TransactionStatus(tid, status, debug, hlocks, wlocks,
top, timeCreated));
}
return new FateStatus(statuses, heldLocks, waitingLocks);
@@ -398,9 +420,10 @@ public class AdminUtil {
FateStatus fateStatus = getStatus(zs, zk, lockPath, filterTxid,
filterStatus);
for (TransactionStatus txStatus : fateStatus.getTransactions()) {
- fmt.format("txid: %s status: %-18s op: %-15s locked: %-15s locking:
%-15s top: %s%n",
+ fmt.format(
+ "txid: %s status: %-18s op: %-15s locked: %-15s locking: %-15s
top: %-15s created: %s%n",
txStatus.getTxid(), txStatus.getStatus(), txStatus.getDebug(),
txStatus.getHeldLocks(),
- txStatus.getWaitingLocks(), txStat
[accumulo] branch main updated (a77edef -> 99cd575)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from a77edef closes #1955 - move log message add 99cd575 Add Validation and Sort function to ZooQueueLock (#1958) No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/fate/AdminUtil.java | 3 +- .../accumulo/fate/zookeeper/ZooQueueLock.java | 54 +- 2 files changed, 55 insertions(+), 2 deletions(-)
[accumulo] branch main updated: Closes #1929. Add property verification for Sampler Options (#1944)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a8db761 Closes #1929. Add property verification for Sampler Options
(#1944)
a8db761 is described below
commit a8db76146740770fe86a84ff5428b15f5c8e7e68
Author: Jeffrey Manno
AuthorDate: Thu Feb 25 13:12:40 2021 -0500
Closes #1929. Add property verification for Sampler Options (#1944)
Co-authored-by: Christopher Tubbs
---
.../core/client/sample/AbstractHashSampler.java| 29 ++
.../core/client/sample/RowColumnSampler.java | 10 ++--
.../accumulo/core/client/sample/Sampler.java | 8 ++
.../accumulo/core/sample/impl/SamplerFactory.java | 13 +++---
4 files changed, 49 insertions(+), 11 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
b/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
index df22797..8f5f29c 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
@@ -23,6 +23,7 @@ import static java.util.Objects.requireNonNull;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Map;
import java.util.Set;
import org.apache.accumulo.core.data.Key;
@@ -57,12 +58,34 @@ public abstract class AbstractHashSampler implements
Sampler {
private int modulus;
private static final Set VALID_OPTIONS = Set.of("hasher", "modulus");
+ private static final Set VALID_VALUES_HASHER = Set.of("murmur3_32",
"md5", "sha1");
+
+ /**
+ * Subclasses with options should override this method to validate subclass
options while also
+ * calling {@code super.validateOptions(config)} to validate base class
options.
+ */
+ @Override
+ public void validateOptions(Map config) {
+for (Map.Entry entry : config.entrySet()) {
+ checkArgument(VALID_OPTIONS.contains(entry.getKey()), "Unknown option:
%s", entry.getKey());
+
+ if (entry.getKey().equals("hasher"))
+checkArgument(VALID_VALUES_HASHER.contains(entry.getValue()),
+"Unknown value for hasher: %s", entry.getValue());
+
+ if (entry.getKey().equals("modulus"))
+checkArgument(Integer.parseInt(entry.getValue()) > 0,
+"Improper Integer value for modulus: %s", entry.getValue());
+}
+ }
/**
* Subclasses with options should override this method and return true if
the option is valid for
* the subclass or if {@code super.isValidOption(opt)} returns true.
+ *
+ * @deprecated since 2.1.0, replaced by {@link #validateOptions(Map)}
*/
-
+ @Deprecated(since = "2.1.0")
protected boolean isValidOption(String option) {
return VALID_OPTIONS.contains(option);
}
@@ -80,10 +103,6 @@ public abstract class AbstractHashSampler implements
Sampler {
requireNonNull(hasherOpt, "Hasher not specified");
requireNonNull(modulusOpt, "Modulus not specified");
-for (String option : config.getOptions().keySet()) {
- checkArgument(isValidOption(option), "Unknown option : %s", option);
-}
-
switch (hasherOpt) {
case "murmur3_32":
hashFunction = Hashing.murmur3_32();
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java
b/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java
index a2785e3..1d71214 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java
@@ -18,8 +18,11 @@
*/
package org.apache.accumulo.core.client.sample;
+import static com.google.common.base.Preconditions.checkArgument;
+
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Map;
import java.util.Set;
import org.apache.accumulo.core.data.ByteSequence;
@@ -80,8 +83,11 @@ public class RowColumnSampler extends AbstractHashSampler {
}
@Override
- protected boolean isValidOption(String option) {
-return super.isValidOption(option) || VALID_OPTIONS.contains(option);
+ public void validateOptions(Map config) {
+super.validateOptions(config);
+for (String option : config.keySet()) {
+ checkArgument(VALID_OPTIONS.contains(option), "Unknown option : %s",
option);
+}
}
@Override
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/sample/Sampler.java
b/core/src/main/java/org/apache/accumulo/core/client/sample/Sampler.java
index d7e07c8..810b8ae 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/samp
[accumulo-testing] branch main updated (689d6d4 -> da57336)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. from 689d6d4 Add a Bulk import to randomwalk MultiTable (#134) add 88ffaac fix build errors add ac69805 add import class exception for ThreadPools add 99c4a3c upgrade accumulo to 2.1.0-SNAPSHOT new da57336 Fixes #135. Fix build errors due recent changes in accumulo and upgrade accumulo version to 2.1.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: contrib/import-control.xml | 2 +- pom.xml | 2 +- .../org/apache/accumulo/testing/randomwalk/Module.java | 4 ++-- .../apache/accumulo/testing/randomwalk/bulk/Setup.java | 6 +++--- .../accumulo/testing/randomwalk/concurrent/Config.java | 16 .../testing/randomwalk/concurrent/Replication.java | 4 ++-- 6 files changed, 17 insertions(+), 17 deletions(-)
[accumulo-testing] 01/01: Fixes #135. Fix build errors due recent changes in accumulo and upgrade accumulo version to 2.1.0
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git commit da5733655dc000bda1703cfbe45065391dcc4067 Merge: 689d6d4 99c4a3c Author: Jeffrey Manno AuthorDate: Thu Feb 11 13:40:30 2021 -0500 Fixes #135. Fix build errors due recent changes in accumulo and upgrade accumulo version to 2.1.0 contrib/import-control.xml | 2 +- pom.xml | 2 +- .../org/apache/accumulo/testing/randomwalk/Module.java | 4 ++-- .../apache/accumulo/testing/randomwalk/bulk/Setup.java | 6 +++--- .../accumulo/testing/randomwalk/concurrent/Config.java | 16 .../testing/randomwalk/concurrent/Replication.java | 4 ++-- 6 files changed, 17 insertions(+), 17 deletions(-)
[accumulo] branch main updated (3edd539 -> ec8c4c6)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 3edd539 Drop debug logs to trace in ZooLock (#1918) add ec8c4c6 Better error message when instance.volumes does not specify the accumulo directory (#1921) No new revisions were added by this update. Summary of changes: core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java | 7 +++ .../src/main/java/org/apache/accumulo/server/fs/VolumeManager.java | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-)
[accumulo] branch main updated (df85cfe -> 8bc4c51)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from df85cfe Merge branch '1.10' into main add 8bc4c51 Explicit error message for grant/revoke System permission (#1906) No new revisions were added by this update. Summary of changes: .../apache/accumulo/shell/commands/GrantCommand.java | 20 +--- .../accumulo/shell/commands/RevokeCommand.java | 20 +--- 2 files changed, 26 insertions(+), 14 deletions(-)
[accumulo] branch 1.10 updated: Remove obsolete TODO (#1901) (#1908)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new 2dacbc3 Remove obsolete TODO (#1901) (#1908)
2dacbc3 is described below
commit 2dacbc33a220cdd989e31f51b662470d4052ad80
Author: Jeffrey Manno
AuthorDate: Thu Feb 4 09:12:48 2021 -0500
Remove obsolete TODO (#1901) (#1908)
---
.../tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java | 2 --
1 file changed, 2 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
index 2faf437..5b539d5 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
@@ -754,8 +754,6 @@ public class InMemoryMap {
int mc = kvCount.get();
MemoryDataSource mds = new MemoryDataSource(iteratorSamplerConfig);
-// TODO seems like a bug that two MemoryDataSources are created... may
need to fix in older
-// branches
SourceSwitchingIterator ssi = new SourceSwitchingIterator(mds);
MemoryIterator mi = new MemoryIterator(new
PartialMutationSkippingIterator(ssi, mc));
mi.setSSI(ssi);
[accumulo] branch main updated (27ed66b -> eb6f5ef)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 27ed66b Merge branch '1.10' into main add eb6f5ef Add userExistsTest to ShellIT (#1894) No new revisions were added by this update. Summary of changes: test/src/main/java/org/apache/accumulo/test/ShellIT.java | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-)
[accumulo] branch main updated (fd70e08 -> 68af47a)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from fd70e08 Closes #637: Add a UUID to the server's ZooLock (#1866) add 68af47a Upgrade to jline3 (#1824) No new revisions were added by this update. Summary of changes: assemble/pom.xml | 10 +- assemble/src/main/assemblies/component.xml | 2 +- assemble/src/main/resources/LICENSE| 68 core/pom.xml | 8 +- .../org/apache/accumulo/core/util/CreateToken.java | 11 +- pom.xml| 10 +- server/base/pom.xml| 8 +- .../apache/accumulo/server/init/Initialize.java| 47 +++--- shell/pom.xml | 8 +- .../main/java/org/apache/accumulo/shell/Shell.java | 171 +++-- .../org/apache/accumulo/shell/ShellCompletor.java | 21 ++- .../shell/commands/AuthenticateCommand.java| 4 +- .../accumulo/shell/commands/ClasspathCommand.java | 14 +- .../accumulo/shell/commands/ClearCommand.java | 18 +-- .../accumulo/shell/commands/ConfigCommand.java | 5 +- .../accumulo/shell/commands/ConstraintCommand.java | 12 +- .../shell/commands/CreateTableCommand.java | 2 +- .../accumulo/shell/commands/CreateUserCommand.java | 4 +- .../apache/accumulo/shell/commands/DUCommand.java | 2 +- .../shell/commands/DeleteNamespaceCommand.java | 2 +- .../accumulo/shell/commands/DeleteRowsCommand.java | 2 +- .../shell/commands/DeleteTableCommand.java | 2 +- .../accumulo/shell/commands/DropUserCommand.java | 22 ++- .../accumulo/shell/commands/GetAuthsCommand.java | 2 +- .../accumulo/shell/commands/GetGroupsCommand.java | 2 +- .../accumulo/shell/commands/HelpCommand.java | 4 +- .../accumulo/shell/commands/HiddenCommand.java | 7 +- .../accumulo/shell/commands/HistoryCommand.java| 10 +- .../accumulo/shell/commands/ListIterCommand.java | 2 +- .../shell/commands/ListShellIterCommand.java | 2 +- .../accumulo/shell/commands/MaxRowCommand.java | 2 +- .../accumulo/shell/commands/MergeCommand.java | 11 +- .../commands/NamespacePermissionsCommand.java | 2 +- .../accumulo/shell/commands/PasswdCommand.java | 6 +- .../accumulo/shell/commands/SetIterCommand.java| 40 ++--- .../commands/ShellPluginConfigurationCommand.java | 4 +- .../shell/commands/SystemPermissionsCommand.java | 2 +- .../accumulo/shell/commands/TableOperation.java| 2 +- .../shell/commands/TablePermissionsCommand.java| 2 +- .../accumulo/shell/commands/TraceCommand.java | 12 +- .../accumulo/shell/commands/UserCommand.java | 2 +- .../shell/commands/UserPermissionsCommand.java | 18 +-- .../accumulo/shell/commands/UsersCommand.java | 2 +- .../accumulo/shell/commands/WhoAmICommand.java | 2 +- .../accumulo/shell/format/DeleterFormatter.java| 49 +++--- .../org/apache/accumulo/shell/ShellConfigTest.java | 13 +- .../shell/commands/DeleteAuthsCommandTest.java | 9 +- .../shell/commands/DropUserCommandTest.java| 12 +- .../shell/commands/HistoryCommandTest.java | 37 +++-- .../shell/commands/SetIterCommandTest.java | 23 +-- .../shell/format/DeleterFormatterTest.java | 22 ++- test/pom.xml | 8 +- .../java/org/apache/accumulo/test/ShellIT.java | 14 +- .../org/apache/accumulo/test/ShellServerIT.java| 15 +- 54 files changed, 417 insertions(+), 374 deletions(-)
[accumulo] branch main updated (bbd87a6 -> 24b4ab4)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from bbd87a6 Closes #1689: Don't catch Throwable (unless it's rethrown) (#1840) add 24b4ab4 Fixes #1867. Move authentication before attempting to create new Client object (#1868) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/accumulo/shell/Shell.java| 19 ++- 1 file changed, 14 insertions(+), 5 deletions(-)
[accumulo] branch 1.10 updated: Throw sercurity exceptions when permissions checks fail. Backport to 1.10 (#1830)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new 56142a8 Throw sercurity exceptions when permissions checks fail.
Backport to 1.10 (#1830)
56142a8 is described below
commit 56142a89952533fef922fa86739a879c073e7c2a
Author: Jeffrey Manno
AuthorDate: Wed Dec 9 10:26:20 2020 -0500
Throw sercurity exceptions when permissions checks fail. Backport to 1.10
(#1830)
* backport sercurity exceptions to 1.10
* fix AuditMessageIT
---
.../accumulo/master/MasterClientServiceHandler.java | 21 ++---
.../org/apache/accumulo/test/AuditMessageIT.java| 2 +-
2 files changed, 15 insertions(+), 8 deletions(-)
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
b/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
index ab45a49..608a819 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
@@ -118,7 +118,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
public long initiateFlush(TInfo tinfo, TCredentials c, String tableId)
throws ThriftSecurityException, ThriftTableOperationException {
String namespaceId = getNamespaceIdFromTableId(TableOperation.FLUSH,
tableId);
-master.security.canFlush(c, tableId, namespaceId);
+if (!master.security.canFlush(c, tableId, namespaceId))
+ throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
String zTablePath = Constants.ZROOT + "/" +
master.getInstance().getInstanceID()
+ Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_FLUSH_ID;
@@ -150,7 +151,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
ByteBuffer endRow, long flushID, long maxLoops)
throws ThriftSecurityException, ThriftTableOperationException {
String namespaceId = getNamespaceIdFromTableId(TableOperation.FLUSH,
tableId);
-master.security.canFlush(c, tableId, namespaceId);
+if (!master.security.canFlush(c, tableId, namespaceId))
+ throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
if (endRow != null && startRow != null
&&
ByteBufferUtil.toText(startRow).compareTo(ByteBufferUtil.toText(endRow)) >= 0)
@@ -305,7 +307,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
@Override
public void shutdown(TInfo info, TCredentials c, boolean stopTabletServers)
throws ThriftSecurityException {
-master.security.canPerformSystemActions(c);
+if (!master.security.canPerformSystemActions(c))
+ throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
if (stopTabletServers) {
master.setMasterGoalState(MasterGoalState.CLEAN_STOP);
EventCoordinator.Listener eventListener = master.nextEvent.getListener();
@@ -319,7 +322,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
@Override
public void shutdownTabletServer(TInfo info, TCredentials c, String
tabletServer, boolean force)
throws ThriftSecurityException {
-master.security.canPerformSystemActions(c);
+if (!master.security.canPerformSystemActions(c))
+ throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
final TServerInstance doomed = master.tserverSet.find(tabletServer);
if (!force) {
@@ -391,7 +395,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
@Override
public void setMasterGoalState(TInfo info, TCredentials c, MasterGoalState
state)
throws ThriftSecurityException {
-master.security.canPerformSystemActions(c);
+if (!master.security.canPerformSystemActions(c))
+ throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
master.setMasterGoalState(state);
}
@@ -399,7 +404,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
@Override
public void removeSystemProperty(TInfo info, TCredentials c, String property)
throws ThriftSecurityException {
-master.security.canPerformSystemActions(c);
+if (!master.security.canPerformSystemActions(c))
+ throw new ThriftSecurityException(c.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
try {
SystemPropUtil.removeSystemProperty(property);
@@ -413,7 +419,8 @@ public class MasterClientServiceHandler extends
FateServiceHandler
@Override
public void setSystemProperty(TInfo info, TCredentials c, String property,
String value)
throws ThriftSecurityException, TException {
-ma
[accumulo-website] branch asf-site updated (2f287ee -> 502c895)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from 2f287ee Automatic Site Publish by Buildbot add 502c895 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 ++-- output/people/index.html | 11 ++- 2 files changed, 8 insertions(+), 7 deletions(-)
[accumulo-website] branch main updated: Move Jeffrey Manno to PMC and Committers (#249)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 4c1392c Move Jeffrey Manno to PMC and Committers (#249)
4c1392c is described below
commit 4c1392cc652365a457f949971d8f169b46737a5b
Author: Jeffrey Manno
AuthorDate: Wed Nov 4 13:32:03 2020 -0500
Move Jeffrey Manno to PMC and Committers (#249)
---
pages/people.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pages/people.md b/pages/people.md
index a5348c7..d8b31cf 100644
--- a/pages/people.md
+++ b/pages/people.md
@@ -41,6 +41,7 @@ $(function() {
| elserj| Josh Elser|
[Hortonworks][HORTONWORKS] | [ET][ET] |
| hkeebler | Holly Keebler | [Arctic
Slope Regional Corp.][ASRC]| [ET][ET] |
| ibella| Ivan Bella| [Arctic
Slope Regional Corp.][ASRC]| [ET][ET] |
+| jmanno| [Jeffrey Manno](https://github.com/Manno15) | [Arctic
Slope Regional Corp.][ASRC]| [ET][ET] |
| jmark99 | [Mark Owens](https://github.com/jmark99) |
| [ET][ET] |
| jtrost| Jason Trost |
[Endgame][ENDGAME] | |
| kturner | [Keith Turner](https://github.com/keith-turner) |
[Microsoft][MICROSOFT] | [ET][ET] |
@@ -103,7 +104,6 @@ GitHub also has a [contributor list][github-contributors]
based on commits.
| Jason Then |
| |
| Jay Shipper |
| |
| Jeff Field | [Cloudera][CLOUDERA]
| [ET][ET] |
-| Jeffrey Manno | [Arctic Slope Regional Corp.][ASRC]
| [ET][ET] |
| Jeffrey S. Schwartz |
| |
| Jeffrey Zeiberg | [Arctic Slope Regional Corp.][ASRC]
| [ET][ET] |
| Jenna Huston|
| [ET][ET] |
[accumulo] branch main updated: Wrap undo in try block (#1759)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5e0bbfc Wrap undo in try block (#1759)
5e0bbfc is described below
commit 5e0bbfcf7fe74a41cc3283d3f255162a5900c671
Author: Jeffrey Manno
AuthorDate: Wed Nov 4 13:25:33 2020 -0500
Wrap undo in try block (#1759)
* wrap undo methods in a try block and add exception handling
---
.../accumulo/master/tableOps/create/ChooseDir.java| 18 +++---
.../accumulo/master/tableOps/create/CreateTable.java | 19 ++-
.../master/tableOps/create/FinishCreateTable.java | 14 +++---
3 files changed, 40 insertions(+), 11 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
b/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
index 4e16f354..e085349 100644
---
a/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
+++
b/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/ChooseDir.java
@@ -35,11 +35,14 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
class ChooseDir extends MasterRepo {
private static final long serialVersionUID = 1L;
private final TableInfo tableInfo;
+ private static final Logger log = LoggerFactory.getLogger(ChooseDir.class);
ChooseDir(TableInfo ti) {
this.tableInfo = ti;
@@ -60,9 +63,18 @@ class ChooseDir extends MasterRepo {
@Override
public void undo(long tid, Master master) throws Exception {
-Path p = tableInfo.getSplitDirsPath();
-FileSystem fs = p.getFileSystem(master.getContext().getHadoopConf());
-fs.delete(p, true);
+// Clean up split files if ChooseDir operation fails
+Path p = null;
+try {
+ if (tableInfo.getInitialSplitSize() > 0) {
+p = tableInfo.getSplitDirsPath();
+FileSystem fs = p.getFileSystem(master.getContext().getHadoopConf());
+fs.delete(p, true);
+ }
+} catch (IOException e) {
+ log.error("Failed to undo ChooseDir operation and failed to clean up
split files at {}", p,
+ e);
+}
}
/**
diff --git
a/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/CreateTable.java
b/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/CreateTable.java
index 061d05f..ca56428 100644
---
a/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/CreateTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/CreateTable.java
@@ -33,9 +33,12 @@ import org.apache.accumulo.master.tableOps.TableInfo;
import org.apache.accumulo.master.tableOps.Utils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CreateTable extends MasterRepo {
private static final long serialVersionUID = 1L;
+ private static final Logger log = LoggerFactory.getLogger(CreateTable.class);
private TableInfo tableInfo;
@@ -82,12 +85,18 @@ public class CreateTable extends MasterRepo {
@Override
public void undo(long tid, Master env) throws IOException {
// Clean up split files if create table operation fails
-if (tableInfo.getInitialSplitSize() > 0) {
- Path p = tableInfo.getSplitPath().getParent();
- FileSystem fs = p.getFileSystem(env.getContext().getHadoopConf());
- fs.delete(p, true);
+Path p = null;
+try {
+ if (tableInfo.getInitialSplitSize() > 0) {
+p = tableInfo.getSplitPath().getParent();
+FileSystem fs = p.getFileSystem(env.getContext().getHadoopConf());
+fs.delete(p, true);
+ }
+} catch (IOException e) {
+ log.error("Table failed to be created and failed to clean up split files
at {}", p, e);
+} finally {
+ Utils.unreserveNamespace(env, tableInfo.getNamespaceId(), tid, false);
}
-Utils.unreserveNamespace(env, tableInfo.getNamespaceId(), tid, false);
}
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
b/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
index aabb739..c8b515e 100644
---
a/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/master/tableOps/create/FinishCreateTable.java
@@ -29,10 +29,13 @@ import org.apache.accumulo.master.tableOps.TableInfo;
import org.apache.accumulo.master.tableOps.Utils;
import org.apache.hadoop.fs.FileSystem;
imp
