[accumulo] branch master updated: Fix #1090 Consolidate/Simplfy ServerOpts (#1330)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 6766d92 Fix #1090 Consolidate/Simplfy ServerOpts (#1330)
6766d92 is described below
commit 6766d925ef258af88fc17cae07cbef6ce4dbec26
Author: hkeebler <49656678+hkeeb...@users.noreply.github.com>
AuthorDate: Thu Sep 19 20:39:31 2019 -0400
Fix #1090 Consolidate/Simplfy ServerOpts (#1330)
* added deprecatedoption
* Fix #1090 Deprecate command option -s in GC
* Fix #1090 Deprecate command -s in GC
removed deprecated option in favor of dropping option
* Fix #1090 Deprecate command -s in GC
includes all suggested changes
---
.../org/apache/accumulo/core/cli/ConfigOpts.java | 50 ++
.../org/apache/accumulo/core/conf/Property.java| 13 ++
server/gc/pom.xml | 4 --
.../apache/accumulo/gc/SimpleGarbageCollector.java | 45 ---
.../gc/SimpleGarbageCollectorOptsTest.java | 38
.../accumulo/gc/SimpleGarbageCollectorTest.java| 1 +
6 files changed, 75 insertions(+), 76 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/cli/ConfigOpts.java
b/core/src/main/java/org/apache/accumulo/core/cli/ConfigOpts.java
index e7dccf9..ca0a234 100644
--- a/core/src/main/java/org/apache/accumulo/core/cli/ConfigOpts.java
+++ b/core/src/main/java/org/apache/accumulo/core/cli/ConfigOpts.java
@@ -45,6 +45,26 @@ public class ConfigOpts extends Help {
return propsPath;
}
+ // catch all for string based dropped options, including those specific to
subclassed extensions
+ // uncomment below if needed
+ // @Parameter(names = {}, hidden=true)
+ private String legacyOpts = null;
+
+ // catch all for boolean dropped options, including those specific to
subclassed extensions
+ @Parameter(names = {"-s", "--safemode"}, hidden = true)
+ private boolean legacyOptsBoolean = false;
+
+ // holds information on dealing with dropped options
+ // option -> Message describing replacement option or property
+ private static Map LEGACY_OPTION_MSG = new HashMap<>();
+ static {
+// garbage collector legacy options
+LEGACY_OPTION_MSG.put("-s", "Replaced by configuration property " +
Property.GC_SAFEMODE);
+LEGACY_OPTION_MSG.put("--safemode",
+"Replaced by configuration property " + Property.GC_SAFEMODE);
+
+ }
+
public static class NullSplitter implements IParameterSplitter {
@Override
public List split(String value) {
@@ -78,17 +98,17 @@ public class ConfigOpts extends Help {
Map config = new HashMap<>();
for (String prop : args) {
String[] propArgs = prop.split("=", 2);
+ String key = propArgs[0].trim();
+ String value;
if (propArgs.length == 2) {
-String key = propArgs[0].trim();
-String value = propArgs[1].trim();
-if (key.isEmpty() || value.isEmpty()) {
- throw new IllegalArgumentException("Invalid command line -o option:
" + prop);
-} else {
- config.put(key, value);
-}
- } else {
+value = propArgs[1].trim();
+ } else { // if a boolean property then it's mere existence assumes true
+value = Property.isValidBooleanPropertyKey(key) ? "true" : "";
+ }
+ if (key.isEmpty() || value.isEmpty()) {
throw new IllegalArgumentException("Invalid command line -o option: "
+ prop);
}
+ config.put(key, value);
}
return config;
}
@@ -96,6 +116,20 @@ public class ConfigOpts extends Help {
@Override
public void parseArgs(String programName, String[] args, Object... others) {
super.parseArgs(programName, args, others);
+if (legacyOpts != null || legacyOptsBoolean) {
+ String errMsg = "";
+ for (String option : args) {
+if (LEGACY_OPTION_MSG.containsKey(option)) {
+ errMsg +=
+ "Option " + option + " has been dropped - " +
LEGACY_OPTION_MSG.get(option) + "\n";
+}
+ }
+ errMsg += "See '-o' property override option";
+ // prints error to console if ran from the command line otherwise there
is no way to know that
+ // an error occurred
+ System.err.println(errMsg);
+ throw new IllegalArgumentException(errMsg);
+}
if (getOverrides().size() > 0) {
log.info("The following configuration was set on the command line:");
for (Map.Entry entry : getOverrides().entrySet()) {
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9f401b7..7be7355 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -523,6 +523,8 @@ public enum Property {
"Do
[accumulo] branch 2.0 updated (d3ff3ae -> 3b627a9)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from d3ff3ae Merge branch '1.9' into 2.0 add af05b27 Add colVis file to ContinuousInputFormat. Fixes #1368 (#1369) new 3b627a9 Merge branch '1.9' into 2.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../test/performance/ContinuousIngest.java | 25 +- 1 file changed, 15 insertions(+), 10 deletions(-)
[accumulo] 01/01: Merge branch '2.0'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit adf2f4bd9e5b3c2cec4f957b1370ef53102f2a64 Merge: 2a76a36 3b627a9 Author: Christopher Tubbs AuthorDate: Thu Sep 19 19:52:25 2019 -0400 Merge branch '2.0'
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3b627a979caf612eea45876e275614311265767b
Merge: d3ff3ae af05b27
Author: Christopher Tubbs
AuthorDate: Thu Sep 19 19:51:51 2019 -0400
Merge branch '1.9' into 2.0
.../test/performance/ContinuousIngest.java | 25 +-
1 file changed, 15 insertions(+), 10 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java
index 3e1d497,000..e46d871
mode 100644,00..100644
---
a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java
+++
b/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java
@@@ -1,272 -1,0 +1,277 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
++import java.io.IOException;
+import java.io.InputStreamReader;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.apache.accumulo.core.cli.ClientOpts;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.trace.TraceUtil;
+import org.apache.accumulo.core.util.FastFormat;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.htrace.TraceScope;
+import org.apache.htrace.wrappers.TraceProxy;
+
+import com.beust.jcommander.Parameter;
+
+public class ContinuousIngest {
+
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ private static List visibilities;
+
+ private static void initVisibilities(ContinuousOpts opts) throws Exception {
+if (opts.visFile == null) {
+ visibilities = Collections.singletonList(new ColumnVisibility());
+ return;
+}
+
- visibilities = new ArrayList<>();
-
- FileSystem fs = FileSystem.get(new Configuration());
- BufferedReader in =
- new BufferedReader(new InputStreamReader(fs.open(new
Path(opts.visFile)), UTF_8));
++visibilities = readVisFromFile(opts.visFile);
++ }
+
- String line;
++ public static List readVisFromFile(String visFile) {
++List vis = new ArrayList<>();
+
- while ((line = in.readLine()) != null) {
- visibilities.add(new ColumnVisibility(line));
++try (BufferedReader in = new BufferedReader(new InputStreamReader(
++FileSystem.get(new Configuration()).open(new Path(visFile)), UTF_8)))
{
++ String line;
++ while ((line = in.readLine()) != null) {
++vis.add(new ColumnVisibility(line));
++ }
++} catch (IOException e) {
++ System.out.println("ERROR reading visFile " + visFile + ": ");
++ e.printStackTrace();
+}
-
- in.close();
++return vis;
+ }
+
+ private static ColumnVisibility getVisibility(Random rand) {
+return visibilities.get(rand.nextInt(visibilities.size()));
+ }
+
+ static class TestOpts extends ClientOpts {
+@Parameter(names = "--table", description = "table to use")
+String tableName = "ci";
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ContinuousOpts opts = new ContinuousOpts();
+TestOpts clientOpts = new TestOpts();
+try (TraceScope clientSpan =
+clientOpts.parseArgsAndTrace(ContinuousIngest.class.getName(), args,
opts)) {
+
+ initVisibilities(opts);
+
+ if (opts.min < 0 || opts.max < 0 ||
[accumulo] branch master updated (2a76a36 -> adf2f4b)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 2a76a36 Merge branch '2.0' add af05b27 Add colVis file to ContinuousInputFormat. Fixes #1368 (#1369) new 3b627a9 Merge branch '1.9' into 2.0 new adf2f4b Merge branch '2.0' The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] branch 1.9 updated: Add colVis file to ContinuousInputFormat. Fixes #1368 (#1369)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new af05b27 Add colVis file to ContinuousInputFormat. Fixes #1368 (#1369)
af05b27 is described below
commit af05b27ae9e2a576907e7d1d96d8360f187434a7
Author: Mike Miller
AuthorDate: Thu Sep 19 18:46:49 2019 -0400
Add colVis file to ContinuousInputFormat. Fixes #1368 (#1369)
---
.../accumulo/test/continuous/ContinuousIngest.java | 25 +-
.../test/continuous/ContinuousInputFormat.java | 17 +--
2 files changed, 30 insertions(+), 12 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
index f1fd4d9..17342bc 100644
---
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
+++
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousIngest.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.continuous;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.BufferedReader;
+import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
@@ -57,19 +58,23 @@ public class ContinuousIngest {
return;
}
-visibilities = new ArrayList<>();
-
-FileSystem fs = FileSystem.get(new Configuration());
-BufferedReader in =
-new BufferedReader(new InputStreamReader(fs.open(new
Path(opts.visFile)), UTF_8));
+visibilities = readVisFromFile(opts.visFile);
+ }
-String line;
+ public static List readVisFromFile(String visFile) {
+List vis = new ArrayList<>();
-while ((line = in.readLine()) != null) {
- visibilities.add(new ColumnVisibility(line));
+try (BufferedReader in = new BufferedReader(new InputStreamReader(
+FileSystem.get(new Configuration()).open(new Path(visFile)), UTF_8))) {
+ String line;
+ while ((line = in.readLine()) != null) {
+vis.add(new ColumnVisibility(line));
+ }
+} catch (IOException e) {
+ System.out.println("ERROR reading visFile " + visFile + ": ");
+ e.printStackTrace();
}
-
-in.close();
+return vis;
}
private static ColumnVisibility getVisibility(Random rand) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousInputFormat.java
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousInputFormat.java
index 6aed14a..f140b71 100644
---
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousInputFormat.java
+++
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousInputFormat.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.zip.CRC32;
@@ -34,6 +35,7 @@ import java.util.zip.Checksum;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -56,6 +58,7 @@ public class ContinuousInputFormat extends
InputFormat {
private static final String PROP_FAM_MAX = "mrbulk.fam.max";
private static final String PROP_QUAL_MAX = "mrbulk.qual.max";
private static final String PROP_CHECKSUM = "mrbulk.checksum";
+ private static final String PROP_VIS_FILE = "mrbulk.vis.file";
private static class RandomSplit extends InputSplit implements Writable {
@Override
@@ -93,6 +96,8 @@ public class ContinuousInputFormat extends
InputFormat {
conf.setLong(PROP_ROW_MAX, opts.max);
conf.setInt(PROP_FAM_MAX, opts.maxColF);
conf.setInt(PROP_QUAL_MAX, opts.maxColQ);
+if (opts.visFile != null)
+ conf.set(PROP_VIS_FILE, opts.visFile);
conf.setBoolean(PROP_CHECKSUM, opts.checksum);
}
@@ -110,6 +115,7 @@ public class ContinuousInputFormat extends
InputFormat {
long maxRow;
int maxFam;
int maxQual;
+ List visibilities;
boolean checksum;
Key prevKey;
@@ -126,6 +132,12 @@ public class ContinuousInputFormat extends
InputFormat {
maxFam = job.getConfiguration().getInt(PROP_FAM_MAX, Short.MAX_VALUE);
maxQual = job.getConfiguration().getInt(PROP_QUAL_MAX,
Short.MAX_VALUE);
checksum = job.getConfiguration().getBoolean(PROP_CHECKSUM, false);
+String visFile = job.getConfiguration().get(PROP_VIS_FILE);
+if (visFile == null) {
+ visibilities = Collections.singletonList(new ColumnVisibility());
+} else {
+ visibilities =
[accumulo-testing] branch master updated: Update ContinuousInputFormat to use col vis
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push:
new 24f4898 Update ContinuousInputFormat to use col vis
24f4898 is described below
commit 24f489805f8232860078a55ce464033bd4783570
Author: Mike Miller
AuthorDate: Thu Sep 19 18:33:26 2019 -0400
Update ContinuousInputFormat to use col vis
---
.../testing/continuous/ContinuousIngest.java | 23 +-
.../testing/continuous/ContinuousInputFormat.java | 10 --
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
index c5229f7..0f6b04d 100644
--- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
+++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java
@@ -97,15 +97,7 @@ public class ContinuousIngest {
try (ContinuousEnv env = new ContinuousEnv(args)) {
- String vis = env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES);
- if (vis == null) {
-visibilities = Collections.singletonList(new ColumnVisibility());
- } else {
-visibilities = new ArrayList<>();
-for (String v : vis.split(",")) {
- visibilities.add(new ColumnVisibility(v.trim()));
-}
- }
+ visibilities =
parseVisibilities(env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES));
long rowMin = env.getRowMin();
long rowMax = env.getRowMax();
@@ -215,6 +207,19 @@ public class ContinuousIngest {
}
}
+ public static List parseVisibilities(String visString) {
+List vis;
+if (visString == null) {
+ vis = Collections.singletonList(new ColumnVisibility());
+} else {
+ vis = new ArrayList<>();
+ for (String v : visString.split(",")) {
+vis.add(new ColumnVisibility(v.trim()));
+ }
+}
+return vis;
+ }
+
private static long flush(BatchWriter bw, long count, final int
flushInterval, long lastFlushTime)
throws MutationsRejectedException {
long t1 = System.currentTimeMillis();
diff --git
a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
index ecf8172..4e17bb7 100644
---
a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
+++
b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousInputFormat.java
@@ -34,6 +34,7 @@ import java.util.zip.Checksum;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.testing.TestProps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
@@ -57,6 +58,7 @@ public class ContinuousInputFormat extends
InputFormat {
private static final String PROP_FAM_MAX = "mrbulk.fam.max";
private static final String PROP_QUAL_MAX = "mrbulk.qual.max";
private static final String PROP_CHECKSUM = "mrbulk.checksum";
+ private static final String PROP_VIS = "mrbulk.vis";
private static class RandomSplit extends InputSplit implements Writable {
@Override
@@ -96,6 +98,7 @@ public class ContinuousInputFormat extends
InputFormat {
conf.setInt(PROP_QUAL_MAX, env.getMaxColQ());
conf.setBoolean(PROP_CHECKSUM,
Boolean.parseBoolean(env.getTestProperty(TestProps.CI_INGEST_CHECKSUM)));
+conf.set(PROP_VIS, env.getTestProperty(TestProps.CI_INGEST_VISIBILITIES));
}
@Override
@@ -112,6 +115,7 @@ public class ContinuousInputFormat extends
InputFormat {
long maxRow;
int maxFam;
int maxQual;
+ List visibilities;
boolean checksum;
Key prevKey;
@@ -128,6 +132,7 @@ public class ContinuousInputFormat extends
InputFormat {
maxFam = job.getConfiguration().getInt(PROP_FAM_MAX, Short.MAX_VALUE);
maxQual = job.getConfiguration().getInt(PROP_QUAL_MAX,
Short.MAX_VALUE);
checksum = job.getConfiguration().getBoolean(PROP_CHECKSUM, false);
+visibilities =
ContinuousIngest.parseVisibilities(job.getConfiguration().get(PROP_VIS));
random = new Random(new SecureRandom().nextLong());
@@ -140,15 +145,16 @@ public class ContinuousInputFormat extends
InputFormat {
byte[] fam = genCol(random.nextInt(maxFam));
byte[] qual = genCol(random.nextInt(maxQual));
+byte[] cv =
visibilities.get(random.nextInt(visibilities.size())).flatten();
if (cksum != null) {
cksum.update(row);
cksum.update(fam);
cksum.update(qual);
- cksum.update(new byte[0]); // TODO col vis
+ cksum.update(cv);
}
-
