This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new ba6d5e1 Split IteratorUtil up into 2 non-API classes (#986)
ba6d5e1 is described below
commit ba6d5e100aff11177e09b5285e42de58f435144a
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Tue Feb 26 17:03:41 2019 -0500
Split IteratorUtil up into 2 non-API classes (#986)
* Moved methods not useful to users to non-API classes
* Created IterConfigUtil and SystemIteratorUtil for internal use
* Cleaned up loadIterators methods and created IterLoad class to assist
* Remove Generics from IterConfigUtil and other minor improvements
---
.../core/client/ClientSideIteratorScanner.java | 12 +-
.../core/client/admin/NewTableConfiguration.java | 4 +-
.../accumulo/core/client/rfile/RFileScanner.java | 17 +-
.../accumulo/core/clientImpl/OfflineIterator.java | 16 +-
.../core/clientImpl/TableOperationsImpl.java | 4 +-
.../apache/accumulo/core/conf/IterConfigUtil.java | 261 ++++++++++++++
.../org/apache/accumulo/core/conf/IterLoad.java | 66 ++++
.../accumulo/core/iterators/IteratorUtil.java | 400 +--------------------
.../accumulo/core/util/SystemIteratorUtil.java | 110 ++++++
.../IterConfigUtilTest.java} | 58 +--
.../accumulo/server/conf/TableConfiguration.java | 12 +-
.../apache/accumulo/master/FateServiceHandler.java | 4 +-
.../accumulo/tserver/ConditionCheckerContext.java | 12 +-
.../apache/accumulo/tserver/tablet/Compactor.java | 6 +-
.../accumulo/tserver/tablet/ScanDataSource.java | 13 +-
.../shell/commands/CreateTableCommand.java | 4 +-
.../test/performance/scan/CollectTabletStats.java | 10 +-
17 files changed, 540 insertions(+), 469 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index c666b1d..f928f6a 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@ -31,6 +31,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.clientImpl.ScannerOptions;
+import org.apache.accumulo.core.conf.IterConfigUtil;
+import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Column;
@@ -40,7 +42,6 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
import org.apache.accumulo.core.iterators.IteratorAdapter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.security.Authorizations;
@@ -247,10 +248,11 @@ public class ClientSideIteratorScanner extends
ScannerOptions implements Scanner
SortedKeyValueIterator<Key,Value> skvi;
try {
- skvi = IteratorUtil.loadIterators(smi, tm.values(),
serverSideIteratorOptions,
- new ClientSideIteratorEnvironment(getSamplerConfiguration() != null,
- getIteratorSamplerConfigurationInternal()),
- false, null);
+ IteratorEnvironment env = new
ClientSideIteratorEnvironment(getSamplerConfiguration() != null,
+ getIteratorSamplerConfigurationInternal());
+ IterLoad iterLoad = new
IterLoad().iters(tm.values()).iterOpts(serverSideIteratorOptions)
+ .iterEnv(env).useAccumuloClassLoader(false);
+ skvi = IterConfigUtil.loadIterators(smi, iterLoad);
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
index 48849a3..f4b1d8e 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
@@ -36,8 +36,8 @@ import
org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.client.summary.Summarizer;
import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
import org.apache.accumulo.core.clientImpl.TableOperationsHelper;
+import org.apache.accumulo.core.conf.IterConfigUtil;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
@@ -168,7 +168,7 @@ public class NewTableConfiguration {
Map<String,String> propertyMap = new HashMap<>();
if (limitVersion)
-
propertyMap.putAll(IteratorUtil.generateInitialTableProperties(limitVersion));
+
propertyMap.putAll(IterConfigUtil.generateInitialTableProperties(limitVersion));
propertyMap.putAll(summarizerProps);
propertyMap.putAll(samplerProps);
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
index 571f335..2eb2ccb 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
@@ -37,6 +37,8 @@ import org.apache.accumulo.core.clientImpl.ScannerOptions;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.IterConfigUtil;
+import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.crypto.CryptoServiceFactory.ClassloaderType;
@@ -52,7 +54,6 @@ import org.apache.accumulo.core.file.rfile.RFile;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
import org.apache.accumulo.core.iterators.IteratorAdapter;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.system.MultiIterator;
@@ -64,6 +65,7 @@ import org.apache.accumulo.core.spi.cache.CacheEntry;
import org.apache.accumulo.core.spi.cache.CacheType;
import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.SystemIteratorUtil;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.io.Text;
@@ -364,17 +366,20 @@ class RFileScanner extends ScannerOptions implements
Scanner {
if (opts.useSystemIterators) {
SortedSet<Column> cols = this.getFetchedColumns();
families = LocalityGroupUtil.families(cols);
- iterator = IteratorUtil.setupSystemScanIterators(iterator, cols,
getAuthorizations(),
+ iterator = SystemIteratorUtil.setupSystemScanIterators(iterator, cols,
getAuthorizations(),
EMPTY_BYTES, tableConf);
}
try {
if (opts.tableConfig != null && opts.tableConfig.size() > 0) {
- iterator = IteratorUtil.loadIterators(IteratorScope.scan, iterator,
null, tableConf,
- serverSideIteratorList, serverSideIteratorOptions, new
IterEnv());
+ IterLoad il = IterConfigUtil.loadIterConf(IteratorScope.scan,
serverSideIteratorList,
+ serverSideIteratorOptions, tableConf);
+ iterator = IterConfigUtil.loadIterators(iterator,
+ il.iterEnv(new IterEnv()).useAccumuloClassLoader(true));
} else {
- iterator = IteratorUtil.loadIterators(iterator,
serverSideIteratorList,
- serverSideIteratorOptions, new IterEnv(), false, null);
+ iterator = IterConfigUtil.loadIterators(iterator,
+ new
IterLoad().iters(serverSideIteratorList).iterOpts(serverSideIteratorOptions)
+ .iterEnv(new IterEnv()).useAccumuloClassLoader(false));
}
} catch (IOException e) {
throw new RuntimeException(e);
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
index 46407f8..843c72c 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
@@ -34,6 +34,8 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.IterConfigUtil;
+import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.data.Key;
@@ -46,7 +48,6 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.system.MultiIterator;
@@ -58,6 +59,7 @@ import
org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.accumulo.core.util.SystemIteratorUtil;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -344,12 +346,14 @@ class OfflineIterator implements
Iterator<Entry<Key,Value>> {
acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
defaultSecurityLabel = cv.getExpression();
- SortedKeyValueIterator<Key,Value> visFilter =
IteratorUtil.setupSystemScanIterators(multiIter,
- new HashSet<>(options.fetchedColumns), authorizations,
defaultSecurityLabel, acuTableConf);
+ SortedKeyValueIterator<Key,Value> visFilter =
SystemIteratorUtil.setupSystemScanIterators(
+ multiIter, new HashSet<>(options.fetchedColumns), authorizations,
defaultSecurityLabel,
+ acuTableConf);
+ IterLoad iterLoad = IterConfigUtil.loadIterConf(IteratorScope.scan,
+ options.serverSideIteratorList, options.serverSideIteratorOptions,
acuTableConf);
- return iterEnv.getTopLevelIterator(
- IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent,
acuTableConf,
- options.serverSideIteratorList, options.serverSideIteratorOptions,
iterEnv, false));
+ return iterEnv.getTopLevelIterator(IterConfigUtil.loadIterators(visFilter,
+ iterLoad.iterEnv(iterEnv).useAccumuloClassLoader(false)));
}
@Override
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 02be225..bf59cc6 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -100,7 +100,6 @@ import org.apache.accumulo.core.dataImpl.thrift.TRowRange;
import org.apache.accumulo.core.dataImpl.thrift.TSummaries;
import org.apache.accumulo.core.dataImpl.thrift.TSummarizerConfiguration;
import org.apache.accumulo.core.dataImpl.thrift.TSummaryRequest;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
@@ -128,6 +127,7 @@ import org.apache.accumulo.core.util.MapCounter;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.SystemIteratorUtil;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.fate.util.Retry;
@@ -838,7 +838,7 @@ public class TableOperationsImpl extends
TableOperationsHelper {
List<ByteBuffer> args =
Arrays.asList(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8)),
start == null ? EMPTY : TextUtil.getByteBuffer(start),
end == null ? EMPTY : TextUtil.getByteBuffer(end),
-
ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(config.getIterators())),
+
ByteBuffer.wrap(SystemIteratorUtil.encodeIteratorSettings(config.getIterators())),
ByteBuffer.wrap(CompactionStrategyConfigUtil.encode(config.getCompactionStrategy())));
Map<String,String> opts = new HashMap<>();
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/IterConfigUtil.java
b/core/src/main/java/org/apache/accumulo/core/conf/IterConfigUtil.java
new file mode 100644
index 0000000..a5fe350
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/conf/IterConfigUtil.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.conf;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.constraints.DefaultKeySizeConstraint;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for configuring iterators. These methods were moved from
IteratorUtil so that it
+ * could be treated as API.
+ */
+public class IterConfigUtil {
+ private static final Logger log =
LoggerFactory.getLogger(IterConfigUtil.class);
+
+ public static final Comparator<IterInfo> ITER_INFO_COMPARATOR = Comparator
+ .comparingInt(IterInfo::getPriority);
+
+ /**
+ * Fetch the correct configuration key prefix for the given scope. Throws an
+ * IllegalArgumentException if no property exists for the given scope.
+ */
+ public static Property getProperty(IteratorScope scope) {
+ requireNonNull(scope);
+ switch (scope) {
+ case scan:
+ return Property.TABLE_ITERATOR_SCAN_PREFIX;
+ case minc:
+ return Property.TABLE_ITERATOR_MINC_PREFIX;
+ case majc:
+ return Property.TABLE_ITERATOR_MAJC_PREFIX;
+ default:
+ throw new IllegalStateException("Could not find configuration property
for IteratorScope");
+ }
+ }
+
+ /**
+ * Generate the initial (default) properties for a table
+ *
+ * @param limitVersion
+ * include a VersioningIterator at priority 20 that retains a
single version of a given
+ * K/V pair.
+ * @return A map of Table properties
+ */
+ public static Map<String,String> generateInitialTableProperties(boolean
limitVersion) {
+ TreeMap<String,String> props = new TreeMap<>();
+
+ if (limitVersion) {
+ for (IteratorScope iterScope : IteratorScope.values()) {
+ props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers",
+ "20," + VersioningIterator.class.getName());
+ props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() +
".vers.opt.maxVersions", "1");
+ }
+ }
+
+ props.put(Property.TABLE_CONSTRAINT_PREFIX + "1",
DefaultKeySizeConstraint.class.getName());
+
+ return props;
+ }
+
+ public static List<IterInfo> parseIterConf(IteratorScope scope,
List<IterInfo> iters,
+ Map<String,Map<String,String>> allOptions, AccumuloConfiguration conf) {
+ Map<String,String> properties =
conf.getAllPropertiesWithPrefix(getProperty(scope));
+ ArrayList<IterInfo> iterators = new ArrayList<>(iters);
+ final Property scopeProperty = getProperty(scope);
+ final String scopePropertyKey = scopeProperty.getKey();
+
+ for (Entry<String,String> entry : properties.entrySet()) {
+ String suffix = entry.getKey().substring(scopePropertyKey.length());
+ String suffixSplit[] = suffix.split("\\.", 3);
+
+ if (suffixSplit.length == 1) {
+ String sa[] = entry.getValue().split(",");
+ int prio = Integer.parseInt(sa[0]);
+ String className = sa[1];
+ iterators.add(new IterInfo(prio, className, suffixSplit[0]));
+ } else if (suffixSplit.length == 3 && suffixSplit[1].equals("opt")) {
+ String iterName = suffixSplit[0];
+ String optName = suffixSplit[2];
+
+ Map<String,String> options = allOptions.get(iterName);
+ if (options == null) {
+ options = new HashMap<>();
+ allOptions.put(iterName, options);
+ }
+
+ options.put(optName, entry.getValue());
+
+ } else {
+ throw new IllegalArgumentException("Invalid iterator format: " +
entry.getKey());
+ }
+ }
+
+ Collections.sort(iterators, ITER_INFO_COMPARATOR);
+ return iterators;
+ }
+
+ public static void mergeIteratorConfig(List<IterInfo> destList,
+ Map<String,Map<String,String>> destOpts, List<IterInfo> tableIters,
+ Map<String,Map<String,String>> tableOpts, List<IterInfo> ssi,
+ Map<String,Map<String,String>> ssio) {
+ destList.addAll(tableIters);
+ destList.addAll(ssi);
+ Collections.sort(destList, ITER_INFO_COMPARATOR);
+
+ Set<Entry<String,Map<String,String>>> es = tableOpts.entrySet();
+ for (Entry<String,Map<String,String>> entry : es) {
+ if (entry.getValue() == null) {
+ destOpts.put(entry.getKey(), null);
+ } else {
+ destOpts.put(entry.getKey(), new HashMap<>(entry.getValue()));
+ }
+ }
+
+ mergeOptions(ssio, destOpts);
+
+ }
+
+ private static void mergeOptions(Map<String,Map<String,String>> ssio,
+ Map<String,Map<String,String>> allOptions) {
+ ssio.forEach((k, v) -> {
+ if (v != null) {
+ Map<String,String> options = allOptions.get(k);
+ if (options == null) {
+ allOptions.put(k, v);
+ } else {
+ options.putAll(v);
+ }
+ }
+ });
+ }
+
+ public static IterLoad loadIterConf(IteratorScope scope, List<IterInfo>
iters,
+ Map<String,Map<String,String>> iterOpts, AccumuloConfiguration conf) {
+ Map<String,Map<String,String>> allOptions = new HashMap<>();
+ List<IterInfo> iterators = parseIterConf(scope, iters, allOptions, conf);
+ mergeOptions(iterOpts, allOptions);
+ return new IterLoad().iters(iterators).iterOpts(allOptions);
+ }
+
+ /**
+ * Convert the list of iterators to IterInfo objects and then load the stack.
+ */
+ public static SortedKeyValueIterator<Key,Value>
convertItersAndLoad(IteratorScope scope,
+ SortedKeyValueIterator<Key,Value> source, AccumuloConfiguration conf,
+ List<IteratorSetting> iterators, IteratorEnvironment env) throws
IOException {
+
+ List<IterInfo> ssiList = new ArrayList<>();
+ Map<String,Map<String,String>> ssio = new HashMap<>();
+
+ for (IteratorSetting is : iterators) {
+ ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(),
is.getName()));
+ ssio.put(is.getName(), is.getOptions());
+ }
+
+ IterLoad il = loadIterConf(scope, ssiList, ssio, conf);
+ il =
il.iterEnv(env).useAccumuloClassLoader(true).context(conf.get(Property.TABLE_CLASSPATH));
+ return loadIterators(source, il);
+ }
+
+ /**
+ * Load a stack of iterators provided in the IterLoad, starting with source.
+ */
+ public static SortedKeyValueIterator<Key,Value> loadIterators(
+ SortedKeyValueIterator<Key,Value> source, IterLoad iterLoad) throws
IOException {
+ SortedKeyValueIterator<Key,Value> prev = source;
+
+ try {
+ for (IterInfo iterInfo : iterLoad.iters) {
+
+ Class<SortedKeyValueIterator<Key,Value>> clazz = null;
+ log.trace("Attempting to load iterator class {}", iterInfo.className);
+ if (iterLoad.classCache != null) {
+ clazz = iterLoad.classCache.get(iterInfo.className);
+
+ if (clazz == null) {
+ clazz = loadClass(iterLoad.useAccumuloClassLoader,
iterLoad.context, iterInfo);
+ iterLoad.classCache.put(iterInfo.className, clazz);
+ }
+ } else {
+ clazz = loadClass(iterLoad.useAccumuloClassLoader, iterLoad.context,
iterInfo);
+ }
+
+ SortedKeyValueIterator<Key,Value> skvi = clazz.newInstance();
+
+ Map<String,String> options = iterLoad.iterOpts.get(iterInfo.iterName);
+
+ if (options == null)
+ options = Collections.emptyMap();
+
+ skvi.init(prev, options, iterLoad.iteratorEnvironment);
+ prev = skvi;
+ }
+ } catch (ClassNotFoundException | IllegalAccessException |
InstantiationException e) {
+ log.error(e.toString());
+ throw new RuntimeException(e);
+ }
+ return prev;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Class<SortedKeyValueIterator<Key,Value>> loadClass(boolean
useAccumuloClassLoader,
+ String context, IterInfo iterInfo) throws ClassNotFoundException,
IOException {
+ Class<SortedKeyValueIterator<Key,Value>> clazz;
+ if (useAccumuloClassLoader) {
+ if (context != null && !context.equals("")) {
+ clazz = (Class<SortedKeyValueIterator<Key,Value>>)
AccumuloVFSClassLoader
+ .getContextManager()
+ .loadClass(context, iterInfo.className,
SortedKeyValueIterator.class);
+ log.trace("Iterator class {} loaded from context {}, classloader: {}",
iterInfo.className,
+ context, clazz.getClassLoader());
+ } else {
+ clazz = (Class<SortedKeyValueIterator<Key,Value>>)
AccumuloVFSClassLoader
+ .loadClass(iterInfo.className, SortedKeyValueIterator.class);
+ log.trace("Iterator class {} loaded from AccumuloVFSClassLoader: {}",
iterInfo.className,
+ clazz.getClassLoader());
+ }
+ } else {
+ clazz = (Class<SortedKeyValueIterator<Key,Value>>)
Class.forName(iterInfo.className)
+ .asSubclass(SortedKeyValueIterator.class);
+ log.trace("Iterator class {} loaded from classpath", iterInfo.className);
+ }
+ return clazz;
+ }
+}
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/IterLoad.java
b/core/src/main/java/org/apache/accumulo/core/conf/IterLoad.java
new file mode 100644
index 0000000..a063b95
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/conf/IterLoad.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.conf;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+public class IterLoad {
+
+ Collection<IterInfo> iters;
+ Map<String,Map<String,String>> iterOpts;
+ IteratorEnvironment iteratorEnvironment;
+ boolean useAccumuloClassLoader;
+ String context;
+ Map<String,Class<SortedKeyValueIterator<Key,Value>>> classCache;
+
+ public IterLoad iters(Collection<IterInfo> iters) {
+ this.iters = iters;
+ return this;
+ }
+
+ public IterLoad iterOpts(Map<String,Map<String,String>> iterOpts) {
+ this.iterOpts = iterOpts;
+ return this;
+ }
+
+ public IterLoad iterEnv(IteratorEnvironment iteratorEnvironment) {
+ this.iteratorEnvironment = iteratorEnvironment;
+ return this;
+ }
+
+ public IterLoad useAccumuloClassLoader(boolean useAccumuloClassLoader) {
+ this.useAccumuloClassLoader = useAccumuloClassLoader;
+ return this;
+ }
+
+ public IterLoad context(String context) {
+ this.context = context;
+ return this;
+ }
+
+ public IterLoad
classCache(Map<String,Class<SortedKeyValueIterator<Key,Value>>> classCache) {
+ this.classCache = classCache;
+ return this;
+ }
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
index 4c473c4..c76b85a 100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
@@ -16,348 +16,25 @@
*/
package org.apache.accumulo.core.iterators;
-import static java.util.Objects.requireNonNull;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.constraints.DefaultKeySizeConstraint;
-import org.apache.accumulo.core.data.Column;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
-import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
-import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
-import org.apache.accumulo.core.iterators.system.DeletingIterator;
-import org.apache.accumulo.core.iterators.system.VisibilityFilter;
-import org.apache.accumulo.core.iterators.user.VersioningIterator;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
-import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting;
-import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+/**
+ * Utility class for Iterators.
+ */
public class IteratorUtil {
-
- private static final Logger log =
LoggerFactory.getLogger(IteratorUtil.class);
-
/**
* Even though this type is not in a public API package, its used by methods
in the public API.
* Therefore it should be treated as public API and should not reference any
non public API types.
* Also this type can not be moved.
*/
- public static enum IteratorScope {
+ public enum IteratorScope {
majc, minc, scan
}
- private static Comparator<IterInfo> ITER_INFO_COMPARATOR = Comparator
- .comparingInt(IterInfo::getPriority);
-
/**
- * Fetch the correct configuration key prefix for the given scope. Throws an
- * IllegalArgumentException if no property exists for the given scope.
+ * Maximize the Start Key timestamp.
*/
- static Property getProperty(IteratorScope scope) {
- requireNonNull(scope);
- switch (scope) {
- case scan:
- return Property.TABLE_ITERATOR_SCAN_PREFIX;
- case minc:
- return Property.TABLE_ITERATOR_MINC_PREFIX;
- case majc:
- return Property.TABLE_ITERATOR_MAJC_PREFIX;
- default:
- throw new IllegalStateException("Could not find configuration property
for IteratorScope");
- }
- }
-
- /**
- * Generate the initial (default) properties for a table
- *
- * @param limitVersion
- * include a VersioningIterator at priority 20 that retains a
single version of a given
- * K/V pair.
- * @return A map of Table properties
- */
- public static Map<String,String> generateInitialTableProperties(boolean
limitVersion) {
- TreeMap<String,String> props = new TreeMap<>();
-
- if (limitVersion) {
- for (IteratorScope iterScope : IteratorScope.values()) {
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers",
- "20," + VersioningIterator.class.getName());
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() +
".vers.opt.maxVersions", "1");
- }
- }
-
- props.put(Property.TABLE_CONSTRAINT_PREFIX + "1",
DefaultKeySizeConstraint.class.getName());
-
- return props;
- }
-
- public static void mergeIteratorConfig(List<IterInfo> destList,
- Map<String,Map<String,String>> destOpts, List<IterInfo> tableIters,
- Map<String,Map<String,String>> tableOpts, List<IterInfo> ssi,
- Map<String,Map<String,String>> ssio) {
- destList.addAll(tableIters);
- destList.addAll(ssi);
- Collections.sort(destList, ITER_INFO_COMPARATOR);
-
- Set<Entry<String,Map<String,String>>> es = tableOpts.entrySet();
- for (Entry<String,Map<String,String>> entry : es) {
- if (entry.getValue() == null) {
- destOpts.put(entry.getKey(), null);
- } else {
- destOpts.put(entry.getKey(), new HashMap<>(entry.getValue()));
- }
- }
-
- IteratorUtil.mergeOptions(ssio, destOpts);
-
- }
-
- public static void parseIterConf(IteratorScope scope, List<IterInfo> iters,
- Map<String,Map<String,String>> allOptions, AccumuloConfiguration conf) {
- final Property scopeProperty = getProperty(scope);
- final String scopePropertyKey = scopeProperty.getKey();
-
- for (Entry<String,String> entry :
conf.getAllPropertiesWithPrefix(scopeProperty).entrySet()) {
- String suffix = entry.getKey().substring(scopePropertyKey.length());
- String suffixSplit[] = suffix.split("\\.", 3);
-
- if (suffixSplit.length == 1) {
- String sa[] = entry.getValue().split(",");
- int prio = Integer.parseInt(sa[0]);
- String className = sa[1];
- iters.add(new IterInfo(prio, className, suffixSplit[0]));
- } else if (suffixSplit.length == 3 && suffixSplit[1].equals("opt")) {
- String iterName = suffixSplit[0];
- String optName = suffixSplit[2];
-
- Map<String,String> options = allOptions.get(iterName);
- if (options == null) {
- options = new HashMap<>();
- allOptions.put(iterName, options);
- }
-
- options.put(optName, entry.getValue());
-
- } else {
- throw new IllegalArgumentException("Invalid iterator format: " +
entry.getKey());
- }
- }
-
- Collections.sort(iters, ITER_INFO_COMPARATOR);
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- IteratorScope scope, SortedKeyValueIterator<K,V> source, KeyExtent
extent,
- AccumuloConfiguration conf, IteratorEnvironment env) throws
IOException {
- List<IterInfo> emptyList = Collections.emptyList();
- Map<String,Map<String,String>> emptyMap = Collections.emptyMap();
- return loadIterators(scope, source, extent, conf, emptyList, emptyMap,
env);
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- IteratorScope scope, SortedKeyValueIterator<K,V> source, KeyExtent
extent,
- AccumuloConfiguration conf, List<IteratorSetting> iterators,
IteratorEnvironment env)
- throws IOException {
-
- List<IterInfo> ssiList = new ArrayList<>();
- Map<String,Map<String,String>> ssio = new HashMap<>();
-
- for (IteratorSetting is : iterators) {
- ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(),
is.getName()));
- ssio.put(is.getName(), is.getOptions());
- }
-
- return loadIterators(scope, source, extent, conf, ssiList, ssio, env,
true);
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- IteratorScope scope, SortedKeyValueIterator<K,V> source, KeyExtent
extent,
- AccumuloConfiguration conf, List<IterInfo> ssiList,
Map<String,Map<String,String>> ssio,
- IteratorEnvironment env) throws IOException {
- return loadIterators(scope, source, extent, conf, ssiList, ssio, env,
true);
- }
-
- private static void parseIteratorConfiguration(IteratorScope scope,
List<IterInfo> iters,
- Map<String,Map<String,String>> ssio, Map<String,Map<String,String>>
allOptions,
- AccumuloConfiguration conf) {
- parseIterConf(scope, iters, allOptions, conf);
-
- mergeOptions(ssio, allOptions);
- }
-
- private static void mergeOptions(Map<String,Map<String,String>> ssio,
- Map<String,Map<String,String>> allOptions) {
- for (Entry<String,Map<String,String>> entry : ssio.entrySet()) {
- if (entry.getValue() == null)
- continue;
- Map<String,String> options = allOptions.get(entry.getKey());
- if (options == null) {
- allOptions.put(entry.getKey(), entry.getValue());
- } else {
- options.putAll(entry.getValue());
- }
- }
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- IteratorScope scope, SortedKeyValueIterator<K,V> source, KeyExtent
extent,
- AccumuloConfiguration conf, List<IterInfo> ssiList,
Map<String,Map<String,String>> ssio,
- IteratorEnvironment env, boolean useAccumuloClassLoader) throws
IOException {
-
- return loadIteratorsHelper(scope, source, conf, ssiList, ssio, env,
useAccumuloClassLoader,
- conf.get(Property.TABLE_CLASSPATH));
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- IteratorScope scope, SortedKeyValueIterator<K,V> source, KeyExtent
extent,
- AccumuloConfiguration conf, List<IterInfo> ssiList,
Map<String,Map<String,String>> ssio,
- IteratorEnvironment env, boolean useAccumuloClassLoader, String
classLoaderContext)
- throws IOException {
-
- return loadIteratorsHelper(scope, source, conf, ssiList, ssio, env,
useAccumuloClassLoader,
- classLoaderContext);
- }
-
- // @formatter:off
- private static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIteratorsHelper(
- // @formatter:on
- IteratorScope scope, SortedKeyValueIterator<K,V> source,
AccumuloConfiguration conf,
- List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
IteratorEnvironment env,
- boolean useAccumuloClassLoader, String classLoaderContext) throws
IOException {
-
- List<IterInfo> iters = new ArrayList<>(ssiList);
- Map<String,Map<String,String>> allOptions = new HashMap<>();
- parseIteratorConfiguration(scope, iters, ssio, allOptions, conf);
- return loadIterators(source, iters, allOptions, env,
useAccumuloClassLoader,
- classLoaderContext);
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- SortedKeyValueIterator<K,V> source, Collection<IterInfo> iters,
- Map<String,Map<String,String>> iterOpts, IteratorEnvironment env,
- boolean useAccumuloClassLoader, String context) throws IOException {
- return loadIterators(source, iters, iterOpts, env, useAccumuloClassLoader,
context, null);
- }
-
- // @formatter:off
- public static <K extends WritableComparable<?>,V extends Writable>
SortedKeyValueIterator<K,V>
- loadIterators(
- // @formatter:on
- SortedKeyValueIterator<K,V> source, Collection<IterInfo> iters,
- Map<String,Map<String,String>> iterOpts, IteratorEnvironment env,
- boolean useAccumuloClassLoader, String context,
- Map<String,Class<? extends SortedKeyValueIterator<K,V>>> classCache)
throws IOException {
- // wrap the source in a SynchronizedIterator in case any of the additional
configured iterators
- // want to use threading
- SortedKeyValueIterator<K,V> prev = source;
-
- try {
- for (IterInfo iterInfo : iters) {
-
- Class<? extends SortedKeyValueIterator<K,V>> clazz = null;
- log.trace("Attempting to load iterator class {}", iterInfo.className);
- if (classCache != null) {
- clazz = classCache.get(iterInfo.className);
-
- if (clazz == null) {
- clazz = loadClass(useAccumuloClassLoader, context, iterInfo);
- classCache.put(iterInfo.className, clazz);
- }
- } else {
- clazz = loadClass(useAccumuloClassLoader, context, iterInfo);
- }
-
- SortedKeyValueIterator<K,V> skvi = clazz.newInstance();
-
- Map<String,String> options = iterOpts.get(iterInfo.iterName);
-
- if (options == null)
- options = Collections.emptyMap();
-
- skvi.init(prev, options, env);
- prev = skvi;
- }
- } catch (ClassNotFoundException | IllegalAccessException |
InstantiationException e) {
- log.error(e.toString());
- throw new RuntimeException(e);
- }
- return prev;
- }
-
- @SuppressWarnings("unchecked")
- // @formatter:off
- private static
- <K extends WritableComparable<?>,V extends Writable> Class<? extends
SortedKeyValueIterator<K,V>>
- loadClass(
- // @formatter:on
- boolean useAccumuloClassLoader, String context, IterInfo iterInfo)
- throws ClassNotFoundException, IOException {
- Class<? extends SortedKeyValueIterator<K,V>> clazz;
- if (useAccumuloClassLoader) {
- if (context != null && !context.equals("")) {
- clazz = (Class<? extends SortedKeyValueIterator<K,V>>)
AccumuloVFSClassLoader
- .getContextManager()
- .loadClass(context, iterInfo.className,
SortedKeyValueIterator.class);
- log.trace("Iterator class {} loaded from context {}, classloader: {}",
iterInfo.className,
- context, clazz.getClassLoader());
- } else {
- clazz = (Class<? extends SortedKeyValueIterator<K,V>>)
AccumuloVFSClassLoader
- .loadClass(iterInfo.className, SortedKeyValueIterator.class);
- log.trace("Iterator class {} loaded from AccumuloVFSClassLoader: {}",
iterInfo.className,
- clazz.getClassLoader());
- }
- } else {
- clazz = (Class<? extends SortedKeyValueIterator<K,V>>)
Class.forName(iterInfo.className)
- .asSubclass(SortedKeyValueIterator.class);
- log.trace("Iterator class {} loaded from classpath", iterInfo.className);
- }
- return clazz;
- }
-
public static Range maximizeStartKeyTimeStamp(Range range) {
Range seekRange = range;
@@ -375,6 +52,9 @@ public class IteratorUtil {
return seekRange;
}
+ /**
+ * Minimize the endKey Timestamp.
+ */
public static Range minimizeEndKeyTimeStamp(Range range) {
Range seekRange = range;
@@ -391,68 +71,4 @@ public class IteratorUtil {
return seekRange;
}
-
- public static TIteratorSetting toTIteratorSetting(IteratorSetting is) {
- return new TIteratorSetting(is.getPriority(), is.getName(),
is.getIteratorClass(),
- is.getOptions());
- }
-
- public static IteratorSetting toIteratorSetting(TIteratorSetting tis) {
- return new IteratorSetting(tis.getPriority(), tis.getName(),
tis.getIteratorClass(),
- tis.getProperties());
- }
-
- public static IteratorConfig toIteratorConfig(List<IteratorSetting>
iterators) {
- ArrayList<TIteratorSetting> tisList = new ArrayList<>();
-
- for (IteratorSetting iteratorSetting : iterators) {
- tisList.add(toTIteratorSetting(iteratorSetting));
- }
-
- return new IteratorConfig(tisList);
- }
-
- public static List<IteratorSetting> toIteratorSettings(IteratorConfig ic) {
- List<IteratorSetting> ret = new ArrayList<>();
- for (TIteratorSetting tIteratorSetting : ic.getIterators()) {
- ret.add(toIteratorSetting(tIteratorSetting));
- }
-
- return ret;
- }
-
- public static byte[] encodeIteratorSettings(IteratorConfig iterators) {
- TSerializer tser = new TSerializer(new TBinaryProtocol.Factory());
-
- try {
- return tser.serialize(iterators);
- } catch (TException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static byte[] encodeIteratorSettings(List<IteratorSetting> iterators)
{
- return encodeIteratorSettings(toIteratorConfig(iterators));
- }
-
- public static List<IteratorSetting> decodeIteratorSettings(byte[] enc) {
- TDeserializer tdser = new TDeserializer(new TBinaryProtocol.Factory());
- IteratorConfig ic = new IteratorConfig();
- try {
- tdser.deserialize(ic, enc);
- } catch (TException e) {
- throw new RuntimeException(e);
- }
- return toIteratorSettings(ic);
- }
-
- public static SortedKeyValueIterator<Key,Value> setupSystemScanIterators(
- SortedKeyValueIterator<Key,Value> source, Set<Column> cols,
Authorizations auths,
- byte[] defaultVisibility, AccumuloConfiguration conf) throws IOException
{
- SortedKeyValueIterator<Key,Value> delIter = DeletingIterator.wrap(source,
false,
- DeletingIterator.getBehavior(conf));
- ColumnFamilySkippingIterator cfsi = new
ColumnFamilySkippingIterator(delIter);
- SortedKeyValueIterator<Key,Value> colFilter =
ColumnQualifierFilter.wrap(cfsi, cols);
- return VisibilityFilter.wrap(colFilter, auths, defaultVisibility);
- }
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/SystemIteratorUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/SystemIteratorUtil.java
new file mode 100644
index 0000000..3e13ad1
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/util/SystemIteratorUtil.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnQualifierFilter;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.VisibilityFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
+import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TBinaryProtocol;
+
+/**
+ * System utility class. Not for client use.
+ */
+public class SystemIteratorUtil {
+
+ public static TIteratorSetting toTIteratorSetting(IteratorSetting is) {
+ return new TIteratorSetting(is.getPriority(), is.getName(),
is.getIteratorClass(),
+ is.getOptions());
+ }
+
+ public static IteratorSetting toIteratorSetting(TIteratorSetting tis) {
+ return new IteratorSetting(tis.getPriority(), tis.getName(),
tis.getIteratorClass(),
+ tis.getProperties());
+ }
+
+ public static IteratorConfig toIteratorConfig(List<IteratorSetting>
iterators) {
+ ArrayList<TIteratorSetting> tisList = new ArrayList<>();
+
+ for (IteratorSetting iteratorSetting : iterators) {
+ tisList.add(toTIteratorSetting(iteratorSetting));
+ }
+
+ return new IteratorConfig(tisList);
+ }
+
+ public static List<IteratorSetting> toIteratorSettings(IteratorConfig ic) {
+ List<IteratorSetting> ret = new ArrayList<>();
+ for (TIteratorSetting tIteratorSetting : ic.getIterators()) {
+ ret.add(toIteratorSetting(tIteratorSetting));
+ }
+
+ return ret;
+ }
+
+ public static byte[] encodeIteratorSettings(IteratorConfig iterators) {
+ TSerializer tser = new TSerializer(new TBinaryProtocol.Factory());
+
+ try {
+ return tser.serialize(iterators);
+ } catch (TException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static byte[] encodeIteratorSettings(List<IteratorSetting> iterators)
{
+ return encodeIteratorSettings(toIteratorConfig(iterators));
+ }
+
+ public static List<IteratorSetting> decodeIteratorSettings(byte[] enc) {
+ TDeserializer tdser = new TDeserializer(new TBinaryProtocol.Factory());
+ IteratorConfig ic = new IteratorConfig();
+ try {
+ tdser.deserialize(ic, enc);
+ } catch (TException e) {
+ throw new RuntimeException(e);
+ }
+ return toIteratorSettings(ic);
+ }
+
+ public static SortedKeyValueIterator<Key,Value> setupSystemScanIterators(
+ SortedKeyValueIterator<Key,Value> source, Set<Column> cols,
Authorizations auths,
+ byte[] defaultVisibility, AccumuloConfiguration conf) throws IOException
{
+ SortedKeyValueIterator<Key,Value> delIter = DeletingIterator.wrap(source,
false,
+ DeletingIterator.getBehavior(conf));
+ ColumnFamilySkippingIterator cfsi = new
ColumnFamilySkippingIterator(delIter);
+ SortedKeyValueIterator<Key,Value> colFilter =
ColumnQualifierFilter.wrap(cfsi, cols);
+ return VisibilityFilter.wrap(colFilter, auths, defaultVisibility);
+ }
+}
diff --git
a/core/src/test/java/org/apache/accumulo/core/iterators/IteratorUtilTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/IterConfigUtilTest.java
similarity index 85%
rename from
core/src/test/java/org/apache/accumulo/core/iterators/IteratorUtilTest.java
rename to
core/src/test/java/org/apache/accumulo/core/conf/IterConfigUtilTest.java
index 497e5c2..ee8359c 100644
---
a/core/src/test/java/org/apache/accumulo/core/iterators/IteratorUtilTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/IterConfigUtilTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.accumulo.core.iterators;
+package org.apache.accumulo.core.conf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -23,22 +23,22 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.ConfigurationCopy;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.SortedMapIterator;
import org.apache.accumulo.core.iterators.system.MultiIteratorTest;
import org.apache.accumulo.core.iterators.user.AgeOffFilter;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
@@ -46,10 +46,11 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class IteratorUtilTest {
+public class IterConfigUtilTest {
- private static final Logger log =
LoggerFactory.getLogger(IteratorUtilTest.class);
+ private static final Logger log =
LoggerFactory.getLogger(IterConfigUtilTest.class);
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new
ArrayList<>();
+ private static final List<IterInfo> EMPTY_ITERS = Collections.emptyList();
static class WrappedIter implements SortedKeyValueIterator<Key,Value> {
@@ -130,6 +131,13 @@ public class IteratorUtilTest {
}
}
+ private SortedKeyValueIterator<Key,Value> createIter(IteratorScope scope,
+ SortedMapIterator source, AccumuloConfiguration conf) throws IOException
{
+ IterLoad iterLoad = IterConfigUtil.loadIterConf(scope, EMPTY_ITERS, new
HashMap<>(), conf);
+ iterLoad = iterLoad.iterEnv(new
DefaultIteratorEnvironment(conf)).useAccumuloClassLoader(true);
+ return IterConfigUtil.loadIterators(source, iterLoad);
+ }
+
@Test
public void test1() throws IOException {
ConfigurationCopy conf = new ConfigurationCopy();
@@ -146,9 +154,7 @@ public class IteratorUtilTest {
MultiIteratorTest.newKeyValue(tm, 2, 0, false, "2");
SortedMapIterator source = new SortedMapIterator(tm);
-
- SortedKeyValueIterator<Key,Value> iter =
IteratorUtil.loadIterators(IteratorScope.minc, source,
- new KeyExtent(TableId.of("tab"), null, null), conf, new
DefaultIteratorEnvironment(conf));
+ SortedKeyValueIterator<Key,Value> iter = createIter(IteratorScope.minc,
source, conf);
iter.seek(new Range(), EMPTY_COL_FAMS, false);
assertTrue(iter.hasTop());
@@ -179,8 +185,7 @@ public class IteratorUtilTest {
SortedMapIterator source = new SortedMapIterator(tm);
- SortedKeyValueIterator<Key,Value> iter =
IteratorUtil.loadIterators(IteratorScope.majc, source,
- new KeyExtent(TableId.of("tab"), null, null), conf, new
DefaultIteratorEnvironment(conf));
+ SortedKeyValueIterator<Key,Value> iter = createIter(IteratorScope.majc,
source, conf);
iter.seek(new Range(), EMPTY_COL_FAMS, false);
assertTrue(iter.hasTop());
@@ -217,8 +222,7 @@ public class IteratorUtilTest {
conf.set(Property.TABLE_ITERATOR_PREFIX + IteratorScope.minc.name() +
".sqIter",
"1," + SquaringIter.class.getName());
- SortedKeyValueIterator<Key,Value> iter =
IteratorUtil.loadIterators(IteratorScope.minc, source,
- new KeyExtent(TableId.of("tab"), null, null), conf, new
DefaultIteratorEnvironment(conf));
+ SortedKeyValueIterator<Key,Value> iter = createIter(IteratorScope.minc,
source, conf);
iter.seek(new Range(), EMPTY_COL_FAMS, false);
assertTrue(iter.hasTop());
@@ -256,8 +260,7 @@ public class IteratorUtilTest {
SortedMapIterator source = new SortedMapIterator(tm);
- SortedKeyValueIterator<Key,Value> iter =
IteratorUtil.loadIterators(IteratorScope.minc, source,
- new KeyExtent(TableId.of("tab"), null, null), conf, new
DefaultIteratorEnvironment(conf));
+ SortedKeyValueIterator<Key,Value> iter = createIter(IteratorScope.minc,
source, conf);
iter.seek(new Range(), EMPTY_COL_FAMS, false);
assertTrue(iter.hasTop());
@@ -294,8 +297,7 @@ public class IteratorUtilTest {
SortedMapIterator source = new SortedMapIterator(tm);
- SortedKeyValueIterator<Key,Value> iter =
IteratorUtil.loadIterators(IteratorScope.minc, source,
- new KeyExtent(TableId.of("tab"), null, null), conf, new
DefaultIteratorEnvironment(conf));
+ SortedKeyValueIterator<Key,Value> iter = createIter(IteratorScope.minc,
source, conf);
iter.seek(new Range(), EMPTY_COL_FAMS, false);
assertTrue(iter.hasTop());
@@ -312,17 +314,15 @@ public class IteratorUtilTest {
// Make some configuration items, one with a bogus scope
data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foo", "50," +
SummingCombiner.class.getName());
- data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foo.opt." +
SummingCombiner.ALL_OPTION, "true");
+ data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foo.opt.all", "true");
data.put(Property.TABLE_ITERATOR_PREFIX + ".fakescope.bar",
"50," + SummingCombiner.class.getName());
data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foo.opt.fakeopt",
"fakevalue");
AccumuloConfiguration conf = new ConfigurationCopy(data);
- List<IterInfo> iterators = new ArrayList<>();
- Map<String,Map<String,String>> options = new HashMap<>();
-
- IteratorUtil.parseIterConf(IteratorScope.scan, iterators, options, conf);
+ List<IterInfo> iterators =
IterConfigUtil.parseIterConf(IteratorScope.scan, EMPTY_ITERS,
+ new HashMap<>(), conf);
assertEquals(1, iterators.size());
IterInfo ii = iterators.get(0);
@@ -337,16 +337,16 @@ public class IteratorUtilTest {
public void testInvalidIteratorFormats() {
Map<String,String> data = new HashMap<>();
- List<IterInfo> iterators = new ArrayList<>();
Map<String,Map<String,String>> options = new HashMap<>();
AccumuloConfiguration conf;
// create iterator with 'dot' in name
+ List<IterInfo> iterators = new ArrayList<>();
try {
data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foo.bar",
"50," + SummingCombiner.class.getName());
conf = new ConfigurationCopy(data);
- IteratorUtil.parseIterConf(IteratorScope.scan, iterators, options, conf);
+ iterators = IterConfigUtil.parseIterConf(IteratorScope.scan, iterators,
options, conf);
} catch (IllegalArgumentException ex) {
log.debug("caught expected exception: " + ex.getMessage());
}
@@ -360,7 +360,7 @@ public class IteratorUtilTest {
data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foo.bar.baz",
"49," + SummingCombiner.class.getName());
conf = new ConfigurationCopy(data);
- IteratorUtil.parseIterConf(IteratorScope.scan, iterators, options, conf);
+ iterators = IterConfigUtil.parseIterConf(IteratorScope.scan, iterators,
options, conf);
} catch (IllegalArgumentException ex) {
log.debug("caught expected exception: " + ex.getMessage());
}
@@ -374,7 +374,7 @@ public class IteratorUtilTest {
"48," + SummingCombiner.class.getName());
data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foobar.opt",
"fakevalue");
conf = new ConfigurationCopy(data);
- IteratorUtil.parseIterConf(IteratorScope.scan, iterators, options, conf);
+ iterators = IterConfigUtil.parseIterConf(IteratorScope.scan, iterators,
options, conf);
assertEquals(1, iterators.size());
IterInfo ii = iterators.get(0);
assertEquals(new IterInfo(48, SummingCombiner.class.getName(),
"foobar"), ii);
@@ -391,7 +391,7 @@ public class IteratorUtilTest {
"47," + SummingCombiner.class.getName());
data.put(Property.TABLE_ITERATOR_SCAN_PREFIX + "foobaz.fake.opt",
"fakevalue");
conf = new ConfigurationCopy(data);
- IteratorUtil.parseIterConf(IteratorScope.scan, iterators, options, conf);
+ iterators = IterConfigUtil.parseIterConf(IteratorScope.scan, iterators,
options, conf);
assertEquals(1, iterators.size());
IterInfo ii = iterators.get(0);
assertEquals(new IterInfo(47, SummingCombiner.class.getName(),
"foobaz"), ii);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
index 5165c5f..085acfa 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
@@ -18,7 +18,7 @@ package org.apache.accumulo.server.conf;
import static java.util.Objects.requireNonNull;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
@@ -29,11 +29,11 @@ import java.util.function.Predicate;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.ConfigurationObserver;
+import org.apache.accumulo.core.conf.IterConfigUtil;
import org.apache.accumulo.core.conf.ObservableConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.accumulo.core.spi.scan.ScanDispatcher;
@@ -205,10 +205,10 @@ public class TableConfiguration extends
ObservableConfiguration {
AtomicReference<ParsedIteratorConfig> ref = iteratorConfig.get(scope);
ParsedIteratorConfig pic = ref.get();
if (pic == null || pic.updateCount != count) {
- List<IterInfo> iters = new ArrayList<>();
- Map<String,Map<String,String>> allOptions = new HashMap<>();
- IteratorUtil.parseIterConf(scope, iters, allOptions, this);
- ParsedIteratorConfig newPic = new ParsedIteratorConfig(iters, allOptions,
+ Map<String,Map<String,String>> allOpts = new HashMap<>();
+ List<IterInfo> iters = IterConfigUtil.parseIterConf(scope,
Collections.emptyList(), allOpts,
+ this);
+ ParsedIteratorConfig newPic = new ParsedIteratorConfig(iters, allOpts,
get(Property.TABLE_CLASSPATH), count);
ref.compareAndSet(pic, newPic);
pic = newPic;
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
index 1f794bf..17a51aa 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
@@ -50,13 +50,13 @@ import
org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
import
org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.master.thrift.BulkImportState;
import org.apache.accumulo.core.master.thrift.FateOperation;
import org.apache.accumulo.core.master.thrift.FateService;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.trace.thrift.TInfo;
import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.SystemIteratorUtil;
import org.apache.accumulo.core.util.Validator;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.fate.ReadOnlyTStore.TStatus;
@@ -441,7 +441,7 @@ class FateServiceHandler implements FateService.Iface {
TableId tableId = validateTableIdArgument(arguments.get(0), tableOp,
null);
byte[] startRow = ByteBufferUtil.toBytes(arguments.get(1));
byte[] endRow = ByteBufferUtil.toBytes(arguments.get(2));
- List<IteratorSetting> iterators = IteratorUtil
+ List<IteratorSetting> iterators = SystemIteratorUtil
.decodeIteratorSettings(ByteBufferUtil.toBytes(arguments.get(3)));
CompactionStrategyConfig compactionStrategy =
CompactionStrategyConfigUtil
.decode(ByteBufferUtil.toBytes(arguments.get(4)));
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
index 9e7c1e2..ecd853c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
@@ -29,6 +29,8 @@ import java.util.Map;
import org.apache.accumulo.core.clientImpl.CompressedIterators;
import org.apache.accumulo.core.clientImpl.CompressedIterators.IterConfig;
+import org.apache.accumulo.core.conf.IterConfigUtil;
+import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
@@ -38,7 +40,6 @@ import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
import org.apache.accumulo.core.dataImpl.thrift.TCMResult;
import org.apache.accumulo.core.dataImpl.thrift.TCMStatus;
import org.apache.accumulo.core.dataImpl.thrift.TCondition;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.server.ServerContext;
@@ -54,7 +55,7 @@ public class ConditionCheckerContext {
private Map<String,Map<String,String>> tableIterOpts;
private TabletIteratorEnvironment tie;
private String context;
- private Map<String,Class<? extends SortedKeyValueIterator<Key,Value>>>
classCache;
+ private Map<String,Class<SortedKeyValueIterator<Key,Value>>> classCache;
private static class MergedIterConfig {
List<IterInfo> mergedIters;
@@ -96,7 +97,7 @@ public class ConditionCheckerContext {
Map<String,Map<String,String>> mergedItersOpts = new HashMap<>(
tableIterOpts.size() + ic.ssio.size());
- IteratorUtil.mergeIteratorConfig(mergedIters, mergedItersOpts,
tableIters, tableIterOpts,
+ IterConfigUtil.mergeIteratorConfig(mergedIters, mergedItersOpts,
tableIters, tableIterOpts,
ic.ssiList, ic.ssio);
mic = new MergedIterConfig(mergedIters, mergedItersOpts);
@@ -104,8 +105,9 @@ public class ConditionCheckerContext {
mergedIterCache.put(key, mic);
}
- return IteratorUtil.loadIterators(systemIter, mic.mergedIters,
mic.mergedItersOpts, tie, true,
- context, classCache);
+ IterLoad iterLoad = new
IterLoad().iters(mic.mergedIters).iterOpts(mic.mergedItersOpts)
+
.iterEnv(tie).useAccumuloClassLoader(true).context(context).classCache(classCache);
+ return IterConfigUtil.loadIterators(systemIter, iterLoad);
}
boolean checkConditions(SortedKeyValueIterator<Key,Value> systemIter,
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
index 05d1861..944aa67 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
@@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.IterConfigUtil;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
@@ -40,7 +41,6 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
@@ -365,8 +365,8 @@ public class Compactor implements Callable<CompactionStats>
{
else
throw new IllegalArgumentException();
- SortedKeyValueIterator<Key,Value> itr =
iterEnv.getTopLevelIterator(IteratorUtil
- .loadIterators(env.getIteratorScope(), cfsi, extent, acuTableConf,
iterators, iterEnv));
+ SortedKeyValueIterator<Key,Value> itr =
iterEnv.getTopLevelIterator(IterConfigUtil
+ .convertItersAndLoad(env.getIteratorScope(), cfsi, acuTableConf,
iterators, iterEnv));
itr.seek(extent.toDataRange(), columnFamilies, inclusive);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
index ca58cb2..913dd68 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
@@ -28,13 +28,14 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.accumulo.core.client.sample.SamplerConfiguration;
+import org.apache.accumulo.core.conf.IterConfigUtil;
+import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.data.Column;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
import org.apache.accumulo.core.iterators.IterationInterruptedException;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
@@ -45,6 +46,7 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.SystemIteratorUtil;
import org.apache.accumulo.server.conf.TableConfiguration.ParsedIteratorConfig;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.tserver.FileManager.ScanFileManager;
@@ -199,7 +201,7 @@ class ScanDataSource implements DataSource {
statsIterator = new StatsIterator(multiIter, TabletServer.seekCount,
tablet.getScannedCounter());
- SortedKeyValueIterator<Key,Value> visFilter =
IteratorUtil.setupSystemScanIterators(
+ SortedKeyValueIterator<Key,Value> visFilter =
SystemIteratorUtil.setupSystemScanIterators(
statsIterator, options.getColumnSet(), options.getAuthorizations(),
options.getDefaultLabels(), tablet.getTableConfiguration());
@@ -221,7 +223,7 @@ class ScanDataSource implements DataSource {
// iterator options.
iterOpts = new HashMap<>(pic.getOpts().size() +
options.getSsio().size());
iterInfos = new ArrayList<>(pic.getIterInfo().size() +
options.getSsiList().size());
- IteratorUtil.mergeIteratorConfig(iterInfos, iterOpts,
pic.getIterInfo(), pic.getOpts(),
+ IterConfigUtil.mergeIteratorConfig(iterInfos, iterOpts,
pic.getIterInfo(), pic.getOpts(),
options.getSsiList(), options.getSsio());
}
@@ -240,8 +242,9 @@ class ScanDataSource implements DataSource {
}
}
- return iterEnv.getTopLevelIterator(
- IteratorUtil.loadIterators(visFilter, iterInfos, iterOpts, iterEnv,
true, context));
+ IterLoad il = new
IterLoad().iters(iterInfos).iterOpts(iterOpts).iterEnv(iterEnv)
+ .useAccumuloClassLoader(true).context(context);
+ return
iterEnv.getTopLevelIterator(IterConfigUtil.loadIterators(visFilter, il));
}
}
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
index 3cc3178..0f54482 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/CreateTableCommand.java
@@ -35,9 +35,9 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.clientImpl.Tables;
+import org.apache.accumulo.core.conf.IterConfigUtil;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.constraints.VisibilityConstraint;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.shell.Shell;
import org.apache.accumulo.shell.Shell.Command;
@@ -136,7 +136,7 @@ public class CreateTableCommand extends Command {
shellState.setTableName(tableName); // switch shell to new table context
if (cl.hasOption(createTableNoDefaultIters.getOpt())) {
- for (String key :
IteratorUtil.generateInitialTableProperties(true).keySet()) {
+ for (String key :
IterConfigUtil.generateInitialTableProperties(true).keySet()) {
shellState.getAccumuloClient().tableOperations().removeProperty(tableName, key);
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
index 515845a..4627e91 100644
---
a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++
b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
@@ -40,6 +40,8 @@ import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.Tables;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.IterConfigUtil;
+import org.apache.accumulo.core.conf.IterLoad;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
@@ -52,7 +54,6 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.SortedMapIterator;
@@ -442,9 +443,10 @@ public class CollectTabletStats {
SortedKeyValueIterator<Key,Value> visFilter =
VisibilityFilter.wrap(colFilter, authorizations,
defaultLabels);
- if (useTableIterators)
- return IteratorUtil.loadIterators(IteratorScope.scan, visFilter, ke,
conf, ssiList, ssio,
- null);
+ if (useTableIterators) {
+ IterLoad il = IterConfigUtil.loadIterConf(IteratorScope.scan, ssiList,
ssio, conf);
+ return IterConfigUtil.loadIterators(visFilter,
il.useAccumuloClassLoader(true));
+ }
return visFilter;
}
