keith-turner closed pull request #378: ACCUMULO-4800 Cache parsing of iterator
config
URL: https://github.com/apache/accumulo/pull/378
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
index 3bf54be6d1..c9b482b450 100644
---
a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
@@ -184,7 +184,8 @@ private void checkType(Property property, PropertyType
type) {
}
/**
- * Each time configurations is changes, this counter should increase.
+ * Each time configuration changes, this counter should increase. Anything
that caches information that is derived from configuration can use this method
to
+ * know when to update.
*/
public long getUpdateCount() {
return 0;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
index 8e68d5fb1c..25a9794074 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
@@ -16,13 +16,22 @@
*/
package org.apache.accumulo.server.conf;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.ConfigurationObserver;
import org.apache.accumulo.core.conf.ObservableConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
@@ -31,6 +40,9 @@
import org.slf4j.LoggerFactory;
import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
public class TableConfiguration extends ObservableConfiguration {
private static final Logger log =
LoggerFactory.getLogger(TableConfiguration.class);
@@ -44,10 +56,17 @@
private final String tableId;
+ private EnumMap<IteratorScope,AtomicReference<ParsedIteratorConfig>>
iteratorConfig;
+
public TableConfiguration(Instance instance, String tableId,
NamespaceConfiguration parent) {
this.instance = instance;
this.tableId = tableId;
this.parent = parent;
+
+ iteratorConfig = new EnumMap<>(IteratorScope.class);
+ for (IteratorScope scope : IteratorScope.values()) {
+ iteratorConfig.put(scope, new
AtomicReference<ParsedIteratorConfig>(null));
+ }
}
void setZooCacheFactory(ZooCacheFactory zcf) {
@@ -144,4 +163,50 @@ public String toString() {
public long getUpdateCount() {
return parent.getUpdateCount() +
getPropCacheAccessor().getZooCache().getUpdateCount();
}
+
+ public static class ParsedIteratorConfig {
+ private final List<IterInfo> tableIters;
+ private final Map<String,Map<String,String>> tableOpts;
+ private final String context;
+ private final long updateCount;
+
+ private ParsedIteratorConfig(List<IterInfo> ii,
Map<String,Map<String,String>> opts, String context, long updateCount) {
+ this.tableIters = ImmutableList.copyOf(ii);
+ Builder<String,Map<String,String>> imb = ImmutableMap.builder();
+ for (Entry<String,Map<String,String>> entry : opts.entrySet()) {
+ imb.put(entry.getKey(), ImmutableMap.copyOf(entry.getValue()));
+ }
+ tableOpts = imb.build();
+ this.context = context;
+ this.updateCount = updateCount;
+ }
+
+ public List<IterInfo> getIterInfo() {
+ return tableIters;
+ }
+
+ public Map<String,Map<String,String>> getOpts() {
+ return tableOpts;
+ }
+
+ public String getContext() {
+ return context;
+ }
+ }
+
+ public ParsedIteratorConfig getParsedIteratorConfig(IteratorScope scope) {
+ long count = getUpdateCount();
+ AtomicReference<ParsedIteratorConfig> ref = iteratorConfig.get(scope);
+ ParsedIteratorConfig pic = ref.get();
+ if (pic == null || pic.updateCount != count) {
+ List<IterInfo> iters = new ArrayList<>();
+ Map<String,Map<String,String>> allOptions = new HashMap<>();
+ IteratorUtil.parseIterConf(scope, iters, allOptions, this);
+ ParsedIteratorConfig newPic = new ParsedIteratorConfig(iters,
allOptions, get(Property.TABLE_CLASSPATH), count);
+ ref.compareAndSet(pic, newPic);
+ pic = newPic;
+ }
+
+ return pic;
+ }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
index b95a82f5b8..06cb1136cb 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ConditionCheckerContext.java
@@ -29,8 +29,6 @@
import org.apache.accumulo.core.client.impl.CompressedIterators;
import org.apache.accumulo.core.client.impl.CompressedIterators.IterConfig;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
@@ -43,6 +41,8 @@
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration.ParsedIteratorConfig;
import org.apache.accumulo.tserver.data.ServerConditionalMutation;
import org.apache.hadoop.io.Text;
@@ -67,16 +67,14 @@
private Map<ByteSequence,MergedIterConfig> mergedIterCache = new HashMap<>();
- ConditionCheckerContext(CompressedIterators compressedIters,
AccumuloConfiguration tableConf) {
+ ConditionCheckerContext(CompressedIterators compressedIters,
TableConfiguration tableConf) {
this.compressedIters = compressedIters;
- tableIters = new ArrayList<>();
- tableIterOpts = new HashMap<>();
+ ParsedIteratorConfig pic =
tableConf.getParsedIteratorConfig(IteratorScope.scan);
- // parse table iterator config once
- IteratorUtil.parseIterConf(IteratorScope.scan, tableIters, tableIterOpts,
tableConf);
-
- context = tableConf.get(Property.TABLE_CLASSPATH);
+ tableIters = pic.getIterInfo();
+ tableIterOpts = pic.getOpts();
+ context = pic.getContext();
classCache = new HashMap<>();
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
index e48d91e11f..ced6eed10d 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -44,6 +45,7 @@
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.server.conf.TableConfiguration.ParsedIteratorConfig;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.tserver.FileManager.ScanFileManager;
import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
@@ -186,12 +188,30 @@ public boolean isCurrent() {
if (!loadIters) {
return visFilter;
- } else if (null == options.getClassLoaderContext()) {
- return
iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan,
visFilter, tablet.getExtent(), tablet.getTableConfiguration(),
- options.getSsiList(), options.getSsio(), iterEnv));
} else {
- return
iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan,
visFilter, tablet.getExtent(), tablet.getTableConfiguration(),
- options.getSsiList(), options.getSsio(), iterEnv, true,
options.getClassLoaderContext()));
+ List<IterInfo> iterInfos;
+ Map<String,Map<String,String>> iterOpts;
+
+ ParsedIteratorConfig pic =
tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
+ if (options.getSsiList().size() == 0 && options.getSsio().size() == 0) {
+ // No scan time iterator options were set, so can just use the
pre-parsed table iterator options.
+ iterInfos = pic.getIterInfo();
+ iterOpts = pic.getOpts();
+ } else {
+ // Scan time iterator options were set, so need to merge those with
pre-parsed table iterator options.
+ iterOpts = new HashMap<>(pic.getOpts().size() +
options.getSsio().size());
+ iterInfos = new ArrayList<>(pic.getIterInfo().size() +
options.getSsiList().size());
+ IteratorUtil.mergeIteratorConfig(iterInfos, iterOpts,
pic.getIterInfo(), pic.getOpts(), options.getSsiList(), options.getSsio());
+ }
+
+ String context;
+ if (options.getClassLoaderContext() != null) {
+ context = options.getClassLoaderContext();
+ } else {
+ context = pic.getContext();
+ }
+
+ return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(visFilter,
iterInfos, iterOpts, iterEnv, true, context));
}
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
