This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push: new e89cd6d Refactor out Non API code (#150) e89cd6d is described below commit e89cd6dbffe6a5031ae65c52035ed818bff1fa38 Author: Dom G <47725857+domgargu...@users.noreply.github.com> AuthorDate: Fri Sep 10 15:06:59 2021 -0400 Refactor out Non API code (#150) Removed or replaced usages of non-API code --- contrib/import-control.xml | 12 ---- .../org/apache/accumulo/testing/TestProps.java | 4 ++ .../testing/continuous/ContinuousIngest.java | 2 +- .../apache/accumulo/testing/ingest/TestIngest.java | 6 +- .../accumulo/testing/ingest/VerifyIngest.java | 13 +---- .../apache/accumulo/testing/merkle/MerkleTree.java | 2 +- .../performance/tests/HighSplitCreationPT.java | 4 +- .../testing/performance/tests/ScanExecutorPT.java | 3 +- .../performance/tests/SplitBalancingPT.java | 5 +- .../testing/performance/util/TestData.java | 2 +- .../testing/randomwalk/concurrent/Replication.java | 8 +-- .../apache/accumulo/testing/util/FastFormat.java | 67 ++++++++++++++++++++++ 12 files changed, 88 insertions(+), 40 deletions(-) diff --git a/contrib/import-control.xml b/contrib/import-control.xml index 840881f..0ca22cc 100644 --- a/contrib/import-control.xml +++ b/contrib/import-control.xml @@ -37,18 +37,6 @@ <allow pkg="org.apache.accumulo.core.conf"/> <allow class="org.apache.accumulo.core.util.threads.ThreadPools"/> - <!-- TODO refactor code to remove the following exceptions --> - <allow class="org.apache.accumulo.core.metadata.MetadataTable"/> - <allow class="org.apache.accumulo.core.replication.ReplicationTable"/> - <allow class="org.apache.accumulo.core.spi.scan.HintScanPrioritizer"/> - <allow class="org.apache.accumulo.core.clientImpl.TabletServerBatchWriter"/> - <allow class="org.apache.accumulo.core.util.FastFormat"/> - <allow class="org.apache.accumulo.core.util.Pair"/> - <allow class="org.apache.accumulo.core.trace.Trace"/> - <allow class="org.apache.accumulo.core.trace.TraceSamplers"/> - <allow class="org.apache.accumulo.core.trace.Span"/> - <allow class="org.apache.accumulo.core.trace.DistributedTrace"/> - <!-- End TODO section --> <!-- disallow everything else coming from accumulo --> <disallow pkg="org.apache.accumulo"/> diff --git a/src/main/java/org/apache/accumulo/testing/TestProps.java b/src/main/java/org/apache/accumulo/testing/TestProps.java index f15ca31..c4c8948 100644 --- a/src/main/java/org/apache/accumulo/testing/TestProps.java +++ b/src/main/java/org/apache/accumulo/testing/TestProps.java @@ -46,6 +46,10 @@ public class TestProps { public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb"; // Number of cores given to each YARN container public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores"; + // Name of metadata table + public static final String METADATA_TABLE_NAME = "accumulo.metadata"; + // Name of replication table + public static final String REPLICATION_TABLE_NAME = "accumulo.replication"; /** Continuous ingest test properties **/ /** Common **/ diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java index 0f6b04d..66f5152 100644 --- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java +++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousIngest.java @@ -38,8 +38,8 @@ import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.util.FastFormat; import org.apache.accumulo.testing.TestProps; +import org.apache.accumulo.testing.util.FastFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java index 4fb90ee..0cd0024 100644 --- a/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java +++ b/src/main/java/org/apache/accumulo/testing/ingest/TestIngest.java @@ -35,7 +35,6 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.rfile.RFile; import org.apache.accumulo.core.client.rfile.RFileWriter; import org.apache.accumulo.core.client.security.SecurityErrorCode; -import org.apache.accumulo.core.clientImpl.TabletServerBatchWriter; import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.data.ConstraintViolationSummary; import org.apache.accumulo.core.data.Key; @@ -44,8 +43,8 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.util.FastFormat; import org.apache.accumulo.testing.cli.ClientOpts; +import org.apache.accumulo.testing.util.FastFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.Text; @@ -184,7 +183,8 @@ public class TestIngest { try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) { if (opts.debug) - Logger.getLogger(TabletServerBatchWriter.class.getName()).setLevel(Level.TRACE); + Logger.getLogger("org.apache.accumulo.core.clientImpl.TabletServerBatchWriter") + .setLevel(Level.TRACE); // test batch update diff --git a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java index b783e8c..d7c36de 100644 --- a/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java +++ b/src/main/java/org/apache/accumulo/testing/ingest/VerifyIngest.java @@ -32,8 +32,6 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -// import org.apache.accumulo.core.trace.DistributedTrace; -// import org.apache.accumulo.core.trace.Trace; import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,18 +60,9 @@ public class VerifyIngest { opts.parseArgs(VerifyIngest.class.getName(), args); try (AccumuloClient client = Accumulo.newClient().from(opts.getClientProps()).build()) { if (opts.trace) { - /* - * String name = VerifyIngest.class.getSimpleName(); DistributedTrace.enable(); - * Trace.on(name); Trace.data("cmdLine", Arrays.asList(args).toString()); - */ + throw new UnsupportedOperationException("Tracing is enabled but not currently supported"); } - verifyIngest(client, opts); - - } finally { - /* - * Trace.off(); DistributedTrace.disable(); - */ } } diff --git a/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java b/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java index cc7f810..a9e213a 100644 --- a/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java +++ b/src/main/java/org/apache/accumulo/testing/merkle/MerkleTree.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.apache.accumulo.core.util.Pair; +import org.apache.commons.math3.util.Pair; import com.google.common.collect.Iterables; diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java index 0678c1f..186fbbf 100644 --- a/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java +++ b/src/main/java/org/apache/accumulo/testing/performance/tests/HighSplitCreationPT.java @@ -22,7 +22,7 @@ import java.util.SortedSet; import java.util.TreeSet; import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.testing.TestProps; import org.apache.accumulo.testing.performance.Environment; import org.apache.accumulo.testing.performance.PerformanceTest; import org.apache.accumulo.testing.performance.Report; @@ -53,7 +53,7 @@ public class HighSplitCreationPT implements PerformanceTest { AccumuloClient client = env.getClient(); client.tableOperations().create(TABLE_NAME); - client.tableOperations().addSplits(MetadataTable.NAME, getMetadataTableSplits()); + client.tableOperations().addSplits(TestProps.METADATA_TABLE_NAME, getMetadataTableSplits()); SortedSet<Text> splits = getTestTableSplits(); diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java index ae9b8c0..e59d480 100644 --- a/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java +++ b/src/main/java/org/apache/accumulo/testing/performance/tests/ScanExecutorPT.java @@ -35,7 +35,6 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.spi.scan.HintScanPrioritizer; import org.apache.accumulo.testing.performance.Environment; import org.apache.accumulo.testing.performance.PerformanceTest; import org.apache.accumulo.testing.performance.Report; @@ -57,7 +56,7 @@ public class ScanExecutorPT implements PerformanceTest { private static final int NUM_QUALS = 10; private static final String SCAN_EXECUTOR_THREADS = "2"; - private static final String SCAN_PRIORITIZER = HintScanPrioritizer.class.getName(); + private static final String SCAN_PRIORITIZER = "org.apache.accumulo.core.spi.scan.HintScanPrioritizer"; private static final String TEST_DESC = "Scan Executor Test. Test running lots of short scans " + "while long scans are running in the background. Each short scan reads a random row and " diff --git a/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java b/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java index c88b241..e36c62e 100644 --- a/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java +++ b/src/main/java/org/apache/accumulo/testing/performance/tests/SplitBalancingPT.java @@ -27,8 +27,8 @@ import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.testing.TestProps; import org.apache.accumulo.testing.performance.Environment; import org.apache.accumulo.testing.performance.PerformanceTest; import org.apache.accumulo.testing.performance.Report; @@ -97,7 +97,8 @@ public class SplitBalancingPT implements PerformanceTest { private Map<String,Integer> getTablets(final AccumuloClient client) { Map<String,Integer> tablets = new HashMap<>(); - try (Scanner scanner = client.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) { + try (Scanner scanner = client.createScanner(TestProps.METADATA_TABLE_NAME, + Authorizations.EMPTY)) { scanner.fetchColumnFamily(TSERVER_ASSIGNED_TABLETS_COL_FAM); Range range = new Range(null, false, RESERVED_PREFIX, false); scanner.setRange(range); diff --git a/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java b/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java index 66a7f7e..e17106e 100644 --- a/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java +++ b/src/main/java/org/apache/accumulo/testing/performance/util/TestData.java @@ -20,7 +20,7 @@ package org.apache.accumulo.testing.performance.util; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.util.FastFormat; +import org.apache.accumulo.testing.util.FastFormat; public class TestData { diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java index 71d8f65..fb775b4 100644 --- a/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java +++ b/src/main/java/org/apache/accumulo/testing/randomwalk/concurrent/Replication.java @@ -46,8 +46,8 @@ import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.replication.ReplicationTable; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.testing.TestProps; import org.apache.accumulo.testing.randomwalk.RandWalkEnv; import org.apache.accumulo.testing.randomwalk.State; import org.apache.accumulo.testing.randomwalk.Test; @@ -80,13 +80,13 @@ public class Replication extends Test { iOps.setProperty(REPLICATION_WORK_PROCESSOR_PERIOD.getKey(), "1s"); // Ensure the replication table is online - ReplicationTable.setOnline(c); - boolean online = ReplicationTable.isOnline(c); + tOps.online(TestProps.REPLICATION_TABLE_NAME, true); + boolean online = tOps.isOnline(TestProps.REPLICATION_TABLE_NAME); for (int i = 0; i < 10; i++) { if (online) break; sleepUninterruptibly(2, TimeUnit.SECONDS); - online = ReplicationTable.isOnline(c); + online = tOps.isOnline(TestProps.REPLICATION_TABLE_NAME); } assertTrue("Replication table was not online", online); diff --git a/src/main/java/org/apache/accumulo/testing/util/FastFormat.java b/src/main/java/org/apache/accumulo/testing/util/FastFormat.java new file mode 100644 index 0000000..541dc7c --- /dev/null +++ b/src/main/java/org/apache/accumulo/testing/util/FastFormat.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.testing.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; + +public class FastFormat { + // Copied over from accumulo + // this 7 to 8 times faster than String.format("%s%06d",prefix, num) + public static byte[] toZeroPaddedString(long num, int width, int radix, byte[] prefix) { + Preconditions.checkArgument(num >= 0); + String strNum = Long.toString(num, radix); + byte[] ret = new byte[Math.max(strNum.length(), width) + prefix.length]; + if (toZeroPaddedString(ret, 0, strNum, width, prefix) != ret.length) + throw new RuntimeException(" Did not format to expected width " + num + " " + width + " " + + radix + " " + new String(prefix, UTF_8)); + return ret; + } + + public static int toZeroPaddedString(byte[] output, int outputOffset, long num, int width, + int radix, byte[] prefix) { + Preconditions.checkArgument(num >= 0); + + String strNum = Long.toString(num, radix); + + return toZeroPaddedString(output, outputOffset, strNum, width, prefix); + } + + private static int toZeroPaddedString(byte[] output, int outputOffset, String strNum, int width, + byte[] prefix) { + + int index = outputOffset; + + for (byte b : prefix) { + output[index++] = b; + } + + int end = width - strNum.length() + index; + + while (index < end) + output[index++] = '0'; + + for (int i = 0; i < strNum.length(); i++) { + output[index++] = (byte) strNum.charAt(i); + } + + return index - outputOffset; + } +}