This is an automated email from the ASF dual-hosted git repository.
domgarguilo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new bec7cc6 Add overloaded Common.createTableWithNamespace() with NTC
(#121)
bec7cc6 is described below
commit bec7cc63bd6561b47a009f5d25975e76cc0e1910
Author: Dom G <domgargu...@apache.org>
AuthorDate: Fri Mar 24 11:00:45 2023 -0400
Add overloaded Common.createTableWithNamespace() with NTC (#121)
* Add overloaded Common.createTableWithNamespace() with NTC
* rename ntc to newTableConfig
---
.../java/org/apache/accumulo/examples/Common.java | 9 ++++++++-
.../apache/accumulo/examples/bloom/BloomFilters.java | 7 +++++--
.../examples/bloom/BloomFiltersNotFound.java | 10 +++++++---
.../org/apache/accumulo/examples/dirlist/Ingest.java | 8 +++++---
.../accumulo/examples/filedata/FileDataIngest.java | 6 +++---
.../accumulo/examples/mapreduce/NGramIngest.java | 20 +++++++++++---------
.../accumulo/examples/mapreduce/WordCount.java | 7 ++++---
.../accumulo/examples/mapreduce/bulk/SetupTable.java | 16 ++++++++--------
8 files changed, 51 insertions(+), 32 deletions(-)
diff --git a/src/main/java/org/apache/accumulo/examples/Common.java
b/src/main/java/org/apache/accumulo/examples/Common.java
index 5fb99df..64131f8 100644
--- a/src/main/java/org/apache/accumulo/examples/Common.java
+++ b/src/main/java/org/apache/accumulo/examples/Common.java
@@ -5,6 +5,7 @@ import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.NamespaceExistsException;
import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +33,12 @@ public class Common {
*/
public static void createTableWithNamespace(final AccumuloClient client,
final String table)
throws AccumuloException, AccumuloSecurityException {
+ createTableWithNamespace(client, table, new NewTableConfiguration());
+ }
+
+ public static void createTableWithNamespace(final AccumuloClient client,
final String table,
+ final NewTableConfiguration newTableConfig)
+ throws AccumuloException, AccumuloSecurityException {
String[] name = table.split("\\.");
if (name.length == 2 && !name[0].isEmpty()) {
try {
@@ -41,7 +48,7 @@ public class Common {
}
}
try {
- client.tableOperations().create(table);
+ client.tableOperations().create(table, newTableConfig);
} catch (TableExistsException e) {
log.warn(TABLE_EXISTS_MSG + table);
}
diff --git a/src/main/java/org/apache/accumulo/examples/bloom/BloomFilters.java
b/src/main/java/org/apache/accumulo/examples/bloom/BloomFilters.java
index d2988f5..bc1001b 100644
--- a/src/main/java/org/apache/accumulo/examples/bloom/BloomFilters.java
+++ b/src/main/java/org/apache/accumulo/examples/bloom/BloomFilters.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.examples.bloom;
+import java.util.Map;
import java.util.Random;
import org.apache.accumulo.core.client.Accumulo;
@@ -25,6 +26,7 @@ import
org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.examples.Common;
@@ -58,8 +60,9 @@ public final class BloomFilters {
private static void createTableAndSetCompactionRatio(AccumuloClient client,
final String tableName) throws AccumuloException,
AccumuloSecurityException {
log.info("Creating {}", tableName);
- Common.createTableWithNamespace(client, tableName);
- client.tableOperations().setProperty(tableName,
"table.compaction.major.ratio", "7");
+ Map<String,String> props = Map.of("table.compaction.major.ratio", "7");
+ var newTableConfig = new NewTableConfiguration().setProperties(props);
+ Common.createTableWithNamespace(client, tableName, newTableConfig);
}
// Write a million rows 3 times flushing files to disk separately
diff --git
a/src/main/java/org/apache/accumulo/examples/bloom/BloomFiltersNotFound.java
b/src/main/java/org/apache/accumulo/examples/bloom/BloomFiltersNotFound.java
index e050102..a36f772 100644
--- a/src/main/java/org/apache/accumulo/examples/bloom/BloomFiltersNotFound.java
+++ b/src/main/java/org/apache/accumulo/examples/bloom/BloomFiltersNotFound.java
@@ -18,11 +18,14 @@ package org.apache.accumulo.examples.bloom;
import static org.apache.accumulo.examples.bloom.BloomFilters.writeData;
+import java.util.Map;
+
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.examples.Common;
import org.apache.accumulo.examples.cli.ClientOpts;
import org.slf4j.Logger;
@@ -38,10 +41,11 @@ public class BloomFiltersNotFound {
opts.parseArgs(BloomFiltersNotFound.class.getName(), args);
try (AccumuloClient client =
Accumulo.newClient().from(opts.getClientPropsPath()).build()) {
+ Map<String,String> props = Map.of(BloomCommon.BLOOM_ENABLED_PROPERTY,
"true");
+ var newTableConfig = new NewTableConfiguration().setProperties(props);
+
Common.createTableWithNamespace(client, BloomCommon.BLOOM_TEST3_TABLE);
- Common.createTableWithNamespace(client, BloomCommon.BLOOM_TEST4_TABLE);
- client.tableOperations().setProperty(BloomCommon.BLOOM_TEST4_TABLE,
- BloomCommon.BLOOM_ENABLED_PROPERTY, "true");
+ Common.createTableWithNamespace(client, BloomCommon.BLOOM_TEST4_TABLE,
newTableConfig);
writeAndFlush(BloomCommon.BLOOM_TEST3_TABLE, client);
writeAndFlush(BloomCommon.BLOOM_TEST4_TABLE, client);
diff --git a/src/main/java/org/apache/accumulo/examples/dirlist/Ingest.java
b/src/main/java/org/apache/accumulo/examples/dirlist/Ingest.java
index daeb96f..ec4e712 100644
--- a/src/main/java/org/apache/accumulo/examples/dirlist/Ingest.java
+++ b/src/main/java/org/apache/accumulo/examples/dirlist/Ingest.java
@@ -26,6 +26,7 @@ import java.util.List;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.lexicoder.Encoder;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -160,11 +161,12 @@ public final class Ingest {
opts.parseArgs(Ingest.class.getName(), args, bwOpts);
try (AccumuloClient client = opts.createAccumuloClient()) {
+ var newTableConfig = new NewTableConfiguration()
+ .attachIterator(new IteratorSetting(1, ChunkCombiner.class));
+
Common.createTableWithNamespace(client, opts.dirTable);
Common.createTableWithNamespace(client, opts.indexTable);
- Common.createTableWithNamespace(client, opts.dataTable);
- client.tableOperations().attachIterator(opts.dataTable,
- new IteratorSetting(1, ChunkCombiner.class));
+ Common.createTableWithNamespace(client, opts.dataTable, newTableConfig);
BatchWriter dirBW = client.createBatchWriter(opts.dirTable,
bwOpts.getBatchWriterConfig());
BatchWriter indexBW = client.createBatchWriter(opts.indexTable,
diff --git
a/src/main/java/org/apache/accumulo/examples/filedata/FileDataIngest.java
b/src/main/java/org/apache/accumulo/examples/filedata/FileDataIngest.java
index 87bc67d..0bca7ec 100644
--- a/src/main/java/org/apache/accumulo/examples/filedata/FileDataIngest.java
+++ b/src/main/java/org/apache/accumulo/examples/filedata/FileDataIngest.java
@@ -28,6 +28,7 @@ import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Mutation;
@@ -198,9 +199,8 @@ public class FileDataIngest {
opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);
try (AccumuloClient client = opts.createAccumuloClient()) {
- Common.createTableWithNamespace(client, opts.getTableName());
- client.tableOperations().attachIterator(opts.getTableName(),
- new IteratorSetting(1, ChunkCombiner.class));
+ Common.createTableWithNamespace(client, opts.getTableName(),
+ new NewTableConfiguration().attachIterator(new IteratorSetting(1,
ChunkCombiner.class)));
try (BatchWriter bw = client.createBatchWriter(opts.getTableName(),
bwOpts.getBatchWriterConfig())) {
diff --git
a/src/main/java/org/apache/accumulo/examples/mapreduce/NGramIngest.java
b/src/main/java/org/apache/accumulo/examples/mapreduce/NGramIngest.java
index b41c240..eabd3ad 100644
--- a/src/main/java/org/apache/accumulo/examples/mapreduce/NGramIngest.java
+++ b/src/main/java/org/apache/accumulo/examples/mapreduce/NGramIngest.java
@@ -19,8 +19,11 @@ package org.apache.accumulo.examples.mapreduce;
import java.io.IOException;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.examples.Common;
@@ -89,18 +92,17 @@ public class NGramIngest {
try (AccumuloClient client = opts.createAccumuloClient()) {
if (!client.tableOperations().exists(opts.tableName)) {
- log.info("Creating table " + opts.tableName);
- Common.createTableWithNamespace(client, opts.tableName);
- SortedSet<Text> splits = new TreeSet<>();
String[] numbers = "1 2 3 4 5 6 7 8 9".split("\\s");
String[] lower = "a b c d e f g h i j k l m n o p q r s t u v w x y
z".split("\\s");
String[] upper = "A B C D E F G H I J K L M N O P Q R S T U V W X Y
Z".split("\\s");
- for (String[] array : new String[][] {numbers, lower, upper}) {
- for (String s : array) {
- splits.add(new Text(s));
- }
- }
- client.tableOperations().addSplits(opts.tableName, splits);
+
+ SortedSet<Text> splits = Stream.of(numbers, lower,
upper).flatMap(Stream::of).map(Text::new)
+ .collect(Collectors.toCollection(TreeSet::new));
+
+ var newTableConfig = new NewTableConfiguration().withSplits(splits);
+
+ log.info("Creating table " + opts.tableName);
+ Common.createTableWithNamespace(client, opts.tableName,
newTableConfig);
}
}
diff --git
a/src/main/java/org/apache/accumulo/examples/mapreduce/WordCount.java
b/src/main/java/org/apache/accumulo/examples/mapreduce/WordCount.java
index d86be5a..7daedc8 100644
--- a/src/main/java/org/apache/accumulo/examples/mapreduce/WordCount.java
+++ b/src/main/java/org/apache/accumulo/examples/mapreduce/WordCount.java
@@ -23,6 +23,7 @@ import java.util.Date;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.examples.Common;
@@ -81,14 +82,14 @@ public final class WordCount {
Opts opts = new Opts();
opts.parseArgs(WordCount.class.getName(), args);
- // Create Accumulo table and attach Summing iterator
+ // Create Accumulo table with Summing iterator attached
try (AccumuloClient client = opts.createAccumuloClient()) {
- Common.createTableWithNamespace(client, opts.tableName);
IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
SummingCombiner.setColumns(is,
Collections.singletonList(new IteratorSetting.Column("count")));
SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
- client.tableOperations().attachIterator(opts.tableName, is);
+ Common.createTableWithNamespace(client, opts.tableName,
+ new NewTableConfiguration().attachIterator(is));
}
// Create M/R job
diff --git
a/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/SetupTable.java
b/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/SetupTable.java
index 259f77b..33ff287 100644
--- a/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/SetupTable.java
+++ b/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/SetupTable.java
@@ -17,12 +17,15 @@
package org.apache.accumulo.examples.mapreduce.bulk;
import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.examples.Common;
import org.apache.accumulo.examples.cli.ClientOpts;
import org.apache.hadoop.io.Text;
@@ -36,19 +39,16 @@ public final class SetupTable {
public static void main(String[] args)
throws AccumuloSecurityException, TableNotFoundException,
AccumuloException {
- final String[] splits = {"row_00000333", "row_00000666"};
+ final Stream<String> splits = Stream.of("row_00000333", "row_00000666");
ClientOpts opts = new ClientOpts();
opts.parseArgs(SetupTable.class.getName(), args);
try (AccumuloClient client =
Accumulo.newClient().from(opts.getClientPropsPath()).build()) {
- Common.createTableWithNamespace(client, BULK_INGEST_TABLE);
-
// create a table with initial partitions
- TreeSet<Text> initialPartitions = new TreeSet<>();
- for (String split : splits) {
- initialPartitions.add(new Text(split));
- }
- client.tableOperations().addSplits(BULK_INGEST_TABLE, initialPartitions);
+ TreeSet<Text> initialPartitions = splits.map(Text::new)
+ .collect(Collectors.toCollection(TreeSet::new));
+ Common.createTableWithNamespace(client, BULK_INGEST_TABLE,
+ new NewTableConfiguration().withSplits(initialPartitions));
}
}
}
