[4/4] accumulo git commit: ACCUMULO-4304 Replace Guava use w/built-ins
ACCUMULO-4304 Replace Guava use w/built-ins Replace functional guava stuff, some predicates, optional, and base-64 classes with Java 8 built-in stuff. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/84f70731 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/84f70731 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/84f70731 Branch: refs/heads/master Commit: 84f7073131c08754877ce4c3b470ed54a4a9aa2a Parents: 04c2984 Author: Christopher TubbsAuthored: Thu May 5 20:05:49 2016 -0400 Committer: Christopher Tubbs Committed: Thu May 5 23:59:03 2016 -0400 -- .../apache/accumulo/core/cli/ClientOpts.java| 6 +- .../core/client/MutationsRejectedException.java | 8 +-- .../core/client/impl/ClientContext.java | 9 ++- .../accumulo/core/client/impl/Credentials.java | 14 ++-- .../accumulo/core/client/impl/Namespaces.java | 6 +- .../core/client/mapreduce/RangeInputSplit.java | 6 +- .../mapreduce/lib/impl/ConfiguratorBase.java| 8 +-- .../mapreduce/lib/impl/InputConfigurator.java | 26 +++ .../lib/partition/RangePartitioner.java | 4 +- .../core/client/mock/MockConfiguration.java | 22 +- .../core/conf/AccumuloConfiguration.java| 27 ++-- .../accumulo/core/conf/ConfigurationCopy.java | 5 +- .../core/conf/DefaultConfiguration.java | 5 +- .../apache/accumulo/core/conf/PropertyType.java | 69 +++--- .../accumulo/core/conf/SiteConfiguration.java | 7 +- .../accumulo/core/data/impl/TabletIdImpl.java | 4 +- .../impl/SeekableByteArrayInputStream.java | 8 +-- .../core/file/rfile/MultiLevelIndex.java| 10 +-- .../iterators/user/IntersectingIterator.java| 10 +-- .../accumulo/core/security/Authorizations.java | 6 +- .../org/apache/accumulo/core/util/Base64.java | 73 .../apache/accumulo/core/util/CreateToken.java | 3 +- .../org/apache/accumulo/core/util/Encoding.java | 9 ++- .../apache/accumulo/core/util/Validator.java| 20 +++--- .../core/util/format/DateFormatSupplier.java| 3 +- .../core/util/format/FormatterConfig.java | 3 +- .../core/client/impl/ClientContextTest.java | 6 +- .../client/mapred/AccumuloInputFormatTest.java | 5 +- .../mapreduce/AccumuloInputFormatTest.java | 5 +- .../lib/impl/ConfiguratorBaseTest.java | 7 +- .../core/conf/DefaultConfigurationTest.java | 6 +- .../core/conf/ObservableConfigurationTest.java | 3 +- .../accumulo/core/conf/PropertyTypeTest.java| 42 --- .../core/conf/SiteConfigurationTest.java| 6 +- .../accumulo/core/util/ValidatorTest.java | 22 +++--- .../examples/simple/mapreduce/RowHash.java | 4 +- .../mapreduce/bulk/BulkIngestExample.java | 6 +- maven-plugin/pom.xml| 6 ++ .../impl/MiniAccumuloClusterImpl.java | 11 +-- .../server/conf/NamespaceConfiguration.java | 7 +- .../server/conf/TableConfiguration.java | 3 +- .../server/conf/ZooCachePropertyAccessor.java | 5 +- .../accumulo/server/conf/ZooConfiguration.java | 5 +- .../server/fs/PreferredVolumeChooser.java | 10 +-- .../server/fs/VolumeChooserEnvironment.java | 2 +- .../accumulo/server/fs/VolumeManager.java | 3 +- .../accumulo/server/fs/VolumeManagerImpl.java | 2 +- .../apache/accumulo/server/fs/VolumeUtil.java | 4 +- .../apache/accumulo/server/init/Initialize.java | 12 ++-- .../server/master/balancer/GroupBalancer.java | 5 +- .../master/balancer/RegexGroupBalancer.java | 3 +- .../master/state/TabletStateChangeIterator.java | 10 +-- .../server/security/SystemCredentials.java | 4 +- .../security/handler/KerberosAuthenticator.java | 12 ++-- .../security/handler/KerberosAuthorizor.java| 14 ++-- .../handler/KerberosPermissionHandler.java | 33 - .../accumulo/server/util/DumpZookeeper.java | 4 +- .../apache/accumulo/server/util/FileUtil.java | 5 +- .../accumulo/server/util/MetadataTableUtil.java | 2 +- .../accumulo/server/util/RandomizeVolumes.java | 3 +- .../accumulo/server/util/RestoreZookeeper.java | 4 +- .../server/conf/NamespaceConfigurationTest.java | 6 +- .../server/conf/TableConfigurationTest.java | 6 +- .../conf/ZooCachePropertyAccessorTest.java | 9 ++- .../server/fs/VolumeManagerImplTest.java| 3 +- .../BaseHostRegexTableLoadBalancerTest.java | 7 +- .../master/balancer/GroupBalancerTest.java | 11 +-- .../HostRegexTableLoadBalancerTest.java | 7 +- ...redentialsUpdatingInvocationHandlerTest.java | 3 +- .../server/security/UserImpersonationTest.java | 2 +- .../accumulo/server/util/FileUtilTest.java | 3 +- .../accumulo/gc/SimpleGarbageCollector.java | 42 ---
[3/4] accumulo git commit: ACCUMULO-4304 Replace Guava use w/built-ins
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84f70731/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java -- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java index 772ba69..7bbfd07 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java @@ -16,6 +16,7 @@ */ package org.apache.accumulo.minicluster.impl; +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.BufferedReader; @@ -109,9 +110,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; -import com.google.common.base.Predicate; import com.google.common.collect.Maps; -import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; /** * This class provides the backing implementation for {@link MiniAccumuloCluster}, and may contain features for internal testing which have not yet been @@ -429,12 +428,8 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster { File clientConfFile = config.getClientConfFile(); // Write only the properties that correspond to ClientConfiguration properties -writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate>() { - @Override - public boolean apply(Entry v) { -return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null; - } -})); +writeConfigProperties(clientConfFile, +Maps.filterEntries(config.getSiteConfig(), v -> ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null)); File siteFile = new File(config.getConfDir(), "accumulo-site.xml"); writeConfig(siteFile, config.getSiteConfig().entrySet()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/84f70731/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java index 3d19723..1389869 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/NamespaceConfiguration.java @@ -17,6 +17,7 @@ package org.apache.accumulo.server.conf; import java.util.Map; +import java.util.function.Predicate; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Instance; @@ -33,8 +34,6 @@ import org.apache.accumulo.server.conf.ZooCachePropertyAccessor.PropCacheKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Predicate; - public class NamespaceConfiguration extends ObservableConfiguration { private static final Logger log = LoggerFactory.getLogger(NamespaceConfiguration.class); @@ -110,10 +109,10 @@ public class NamespaceConfiguration extends ObservableConfiguration { } @Override -public boolean apply(String key) { +public boolean test(String key) { if (isIteratorOrConstraint(key)) return false; - return userFilter.apply(key); + return userFilter.test(key); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/84f70731/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java index d4712b9..08726c3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java @@ -17,6 +17,7 @@ package org.apache.accumulo.server.conf; import java.util.Map; +import java.util.function.Predicate; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Instance; @@ -30,8 +31,6 @@ import org.apache.accumulo.server.conf.ZooCachePropertyAccessor.PropCacheKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Predicate; - public class TableConfiguration extends ObservableConfiguration { private static final Logger log = LoggerFactory.getLogger(TableConfiguration.class);
[1/4] accumulo git commit: ACCUMULO-4299 Update plugin versions for jdk8
Repository: accumulo Updated Branches: refs/heads/master fa80ae12b -> 84f707313 ACCUMULO-4299 Update plugin versions for jdk8 Update all plugins, even if they required java 8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/04c2984e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/04c2984e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/04c2984e Branch: refs/heads/master Commit: 04c2984e24a5a414c1543646000cc03e8c3173ac Parents: fa80ae1 Author: Christopher TubbsAuthored: Thu May 5 17:02:31 2016 -0400 Committer: Christopher Tubbs Committed: Thu May 5 23:58:48 2016 -0400 -- assemble/src/main/assemblies/component.xml | 16 ++-- pom.xml| 118 +--- start/pom.xml | 1 + test/pom.xml | 7 -- 4 files changed, 93 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/04c2984e/assemble/src/main/assemblies/component.xml -- diff --git a/assemble/src/main/assemblies/component.xml b/assemble/src/main/assemblies/component.xml index f9cb4db..22a5c22 100644 --- a/assemble/src/main/assemblies/component.xml +++ b/assemble/src/main/assemblies/component.xml @@ -73,7 +73,7 @@ ../examples/simple - /examples/simple + examples/simple 0755 0644 @@ -82,7 +82,7 @@ ./ - /lib/ext + lib/ext 0755 */** @@ -90,7 +90,7 @@ ./ - /logs + logs 0700 */** @@ -98,7 +98,7 @@ ../docs/src/main/resources - /docs + docs 0755 0644 @@ -109,7 +109,7 @@ ../docs/target - /docs + docs 0755 0644 @@ -135,7 +135,7 @@ conf/templates - /conf + conf 0755 0644 @@ -144,7 +144,7 @@ target/example-configs - /conf/examples + conf/examples 0755 0755 @@ -153,7 +153,7 @@ target/example-configs - /conf/examples + conf/examples 0755 0644 http://git-wip-us.apache.org/repos/asf/accumulo/blob/04c2984e/pom.xml -- diff --git a/pom.xml b/pom.xml index 418ed80..2497daa 100644 --- a/pom.xml +++ b/pom.xml @@ -558,6 +558,11 @@ + org.codehaus.mojo + versions-maven-plugin + 2.2 + + org.gaul modernizer-maven-plugin 1.4.0 @@ -586,7 +591,7 @@ com.github.ekryd.sortpom sortpom-maven-plugin - 2.4.0 + 2.5.0 recommended_2008_06 false @@ -601,7 +606,7 @@ com.github.koraktor mavanagaiata - 0.7.1 + 0.7.2 true @@ -614,11 +619,43 @@ org.apache.maven.plugins maven-shade-plugin - 2.3 + 2.4.3 + + + org.apache.maven.plugins + maven-invoker-plugin + + 2.0.0 + + + org.apache.maven.plugins + maven-source-plugin + + 3.0.0 + + + org.apache.maven.plugins + maven-dependency-plugin + + 2.10 + + + org.apache.maven.plugins + maven-gpg-plugin + + 1.6 + + + org.apache.maven.plugins + maven-scm-plugin + + 1.9.4 org.apache.maven.plugins maven-assembly-plugin + + 2.6 false @@ -641,7 +678,10 @@ + org.apache.maven.plugins maven-clean-plugin + + 3.0.0 @@ -655,7 +695,10 @@ + org.apache.maven.plugins maven-compiler-plugin + + 3.5.1 true true @@ -671,6 +714,8 @@ org.apache.maven.plugins maven-jar-plugin + + 2.6 @@ -683,6 +728,8 @@ org.apache.maven.plugins maven-javadoc-plugin + + 2.10.3 ${project.reporting.outputEncoding} true @@ -694,6 +741,8 @@
[2/4] accumulo git commit: ACCUMULO-4304 Replace Guava use w/built-ins
http://git-wip-us.apache.org/repos/asf/accumulo/blob/84f70731/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java -- diff --git a/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java b/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java index aa74556..871dc79 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java +++ b/shell/src/main/java/org/apache/accumulo/shell/ShellUtil.java @@ -21,17 +21,17 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.FileNotFoundException; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Scanner; -import org.apache.accumulo.core.util.Base64; import org.apache.hadoop.io.Text; public class ShellUtil { /** * Scans the given file line-by-line (ignoring empty lines) and returns a list containing those lines. If decode is set to true, every line is decoded using - * {@link Base64#decodeBase64(byte[])} from the UTF-8 bytes of that line before inserting in the list. + * {@link Base64} from the UTF-8 bytes of that line before inserting in the list. * * @param filename * Path to the file that needs to be scanned @@ -49,7 +49,7 @@ public class ShellUtil { while (file.hasNextLine()) { line = file.nextLine(); if (!line.isEmpty()) { - result.add(decode ? new Text(Base64.decodeBase64(line.getBytes(UTF_8))) : new Text(line)); + result.add(decode ? new Text(Base64.getDecoder().decode(line.getBytes(UTF_8))) : new Text(line)); } } } finally { http://git-wip-us.apache.org/repos/asf/accumulo/blob/84f70731/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java -- diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java index e5d6c59..ee39c7c 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/FateCommand.java @@ -16,10 +16,12 @@ */ package org.apache.accumulo.shell.commands; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.lang.reflect.Type; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.EnumSet; import java.util.Formatter; @@ -33,7 +35,6 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.SiteConfiguration; -import org.apache.accumulo.core.util.Base64; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.AdminUtil; import org.apache.accumulo.fate.ReadOnlyRepo; @@ -84,8 +85,8 @@ public class FateCommand extends Command { public String asBase64; ByteArrayContainer(byte[] ba) { - asUtf8 = new String(ba, StandardCharsets.UTF_8); - asBase64 = Base64.encodeBase64URLSafeString(ba); + asUtf8 = new String(ba, UTF_8); + asBase64 = new String(Base64.getUrlEncoder().encode(ba), UTF_8); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/84f70731/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java -- diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java index 17b7db4..4997242 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java @@ -16,9 +16,12 @@ */ package org.apache.accumulo.shell.commands; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Base64; import java.util.Iterator; import java.util.Map.Entry; @@ -34,7 +37,6 @@ import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.util.Base64; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.format.DefaultFormatter; import org.apache.accumulo.shell.Shell; @@ -103,7 +105,8 @@ public class GetSplitsCommand extends Command { return null; } final int length = text.getLength(); -return encode ? Base64.encodeBase64String(TextUtil.getBytes(text))
[2/3] accumulo git commit: ACCUMULO-4229 SyncingTabletLocator ensures caching clients align with tserver
ACCUMULO-4229 SyncingTabletLocator ensures caching clients align with tserver The previously failing test case now passes. A boolean indicator variable in TabletLocator is set to false when the static TabletLocator cache is cleared. Client writers and readers should check the indicator before using a locally cached Locator. A SyncingTabletLocator automatically fetches the current TabletLocator when the cached TabletLocator becomes outdated (when isValid flips false). Clients now stay in sync with the static TabletLocator cache through the use of SyncingTabletLocator, which TimeoutTabletLocator now extends. Variables assumed invariant for SyncingTabletLocator are now final. This does not affect semantics. The new design allows for TabletServerBatchWriter to use its local Locator cache more aggressively than it used to. The invalidateCache() methods of SyncingTabletLocator defensively sync, guarding against goofy code that might use more than one TabletLocator. It may be possible to replace the static locators collection with a Guava class. This is not done here for simplicity. Closes apache/accumulo#96 Closes apache/accumulo#97 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aac9c0f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aac9c0f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aac9c0f Branch: refs/heads/master Commit: 9aac9c0f011e3b5e188a41956b01cfea15ffa9c5 Parents: 5275f33 Author: Dylan HutchisonAuthored: Sat Apr 23 01:33:24 2016 -0700 Committer: Dylan Hutchison Committed: Thu May 5 14:15:24 2016 -0700 -- .../core/client/impl/ConditionalWriterImpl.java | 4 +- .../core/client/impl/SyncingTabletLocator.java | 115 .../core/client/impl/TabletLocator.java | 12 + .../impl/TabletServerBatchReaderIterator.java | 4 +- .../client/impl/TabletServerBatchWriter.java| 9 +- .../core/client/impl/TimeoutTabletLocator.java | 48 +--- .../test/BatchWriterInTabletServerIT.java | 126 + .../accumulo/test/BatchWriterIterator.java | 49 ++-- .../apache/accumulo/test/SerializationUtil.java | 261 --- .../accumulo/test/util/SerializationUtil.java | 257 ++ .../test/BatchWriterInTabletServerIT.java | 118 - 11 files changed, 557 insertions(+), 446 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aac9c0f/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java index 00140e9..1c1aaf3 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java @@ -113,7 +113,7 @@ class ConditionalWriterImpl implements ConditionalWriter { private Map cache = Collections.synchronizedMap(new LRUMap(1000)); private final ClientContext context; private TabletLocator locator; - private String tableId; + private final String tableId; private long timeout; private final Durability durability; private final String classLoaderContext; @@ -388,7 +388,7 @@ class ConditionalWriterImpl implements ConditionalWriter { this.auths = config.getAuthorizations(); this.ve = new VisibilityEvaluator(config.getAuthorizations()); this.threadPool = new ScheduledThreadPoolExecutor(config.getMaxWriteThreads(), new NamingThreadFactory(this.getClass().getSimpleName())); -this.locator = TabletLocator.getLocator(context, tableId); +this.locator = new SyncingTabletLocator(context, tableId); this.serverQueues = new HashMap (); this.tableId = tableId; this.timeout = config.getTimeout(TimeUnit.MILLISECONDS); http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aac9c0f/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java new file mode 100644 index 000..6e7e072 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information
[3/3] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fa80ae12 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fa80ae12 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fa80ae12 Branch: refs/heads/master Commit: fa80ae12b96e7571ae9424bef052ee450cc5b5e4 Parents: a407333 9aac9c0 Author: Dylan HutchisonAuthored: Thu May 5 14:24:43 2016 -0700 Committer: Dylan Hutchison Committed: Thu May 5 14:24:43 2016 -0700 -- .../core/client/impl/ConditionalWriterImpl.java | 4 +- .../core/client/impl/SyncingTabletLocator.java | 115 .../core/client/impl/TabletLocator.java | 12 + .../impl/TabletServerBatchReaderIterator.java | 4 +- .../client/impl/TabletServerBatchWriter.java| 9 +- .../core/client/impl/TimeoutTabletLocator.java | 48 +--- .../test/BatchWriterInTabletServerIT.java | 126 + .../accumulo/test/BatchWriterIterator.java | 261 +++ .../accumulo/test/util/SerializationUtil.java | 257 ++ 9 files changed, 791 insertions(+), 45 deletions(-) --
[1/3] accumulo git commit: ACCUMULO-4229 Add breaking test to reproduce sync problem
Repository: accumulo Updated Branches: refs/heads/master a40733374 -> fa80ae12b ACCUMULO-4229 Add breaking test to reproduce sync problem Add two tests in BatchWriterInTabletServerIT. The first passes and establishes a baseline. The second fails, proving the issue. I added an extra iterator and a utility class for the IT. These are in the test package, so they do not get packaged into normal builds. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5275f335 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5275f335 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5275f335 Branch: refs/heads/master Commit: 5275f335cf6ba22a017edaccf64e096f7d59ceb2 Parents: c044127 Author: Dylan HutchisonAuthored: Sat Apr 23 01:02:01 2016 -0700 Committer: Dylan Hutchison Committed: Thu May 5 14:14:40 2016 -0700 -- .../accumulo/test/BatchWriterIterator.java | 256 ++ .../apache/accumulo/test/SerializationUtil.java | 261 +++ .../test/BatchWriterInTabletServerIT.java | 118 + 3 files changed, 635 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/5275f335/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java b/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java new file mode 100644 index 000..7d44729 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.TimedOutException; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.impl.TabletLocator; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iterators.WrappingIterator; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; + +/** + * Iterator that opens a BatchWriter and writes to another table. + * + * For each entry passed to this iterator, this writes a certain number of entries with the same key to another table and passes the entry downstream of this + * iterator with its value replaced by either "{@value SUCCESS_STRING}" or a description of what failed. Success counts as all entries writing to the result + * table within a timeout period. Failure counts as one of the entries taking longer than the timeout period. + * + * Configure this iterator by calling the static {@link #iteratorSetting} method. + */ +public class BatchWriterIterator extends WrappingIterator { + private static final Logger log =
[2/2] accumulo git commit: ACCUMULO-4229 SyncingTabletLocator ensures caching clients align with tserver
ACCUMULO-4229 SyncingTabletLocator ensures caching clients align with tserver The previously failing test case now passes. A boolean indicator variable in TabletLocator is set to false when the static TabletLocator cache is cleared. Client writers and readers should check the indicator before using a locally cached Locator. A SyncingTabletLocator automatically fetches the current TabletLocator when the cached TabletLocator becomes outdated (when isValid flips false). Clients now stay in sync with the static TabletLocator cache through the use of SyncingTabletLocator, which TimeoutTabletLocator now extends. Variables assumed invariant for SyncingTabletLocator are now final. This does not affect semantics. The new design allows for TabletServerBatchWriter to use its local Locator cache more aggressively than it used to. The invalidateCache() methods of SyncingTabletLocator defensively sync, guarding against goofy code that might use more than one TabletLocator. It may be possible to replace the static locators collection with a Guava class. This is not done here for simplicity. Closes apache/accumulo#96 Closes apache/accumulo#97 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aac9c0f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aac9c0f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aac9c0f Branch: refs/heads/1.8 Commit: 9aac9c0f011e3b5e188a41956b01cfea15ffa9c5 Parents: 5275f33 Author: Dylan HutchisonAuthored: Sat Apr 23 01:33:24 2016 -0700 Committer: Dylan Hutchison Committed: Thu May 5 14:15:24 2016 -0700 -- .../core/client/impl/ConditionalWriterImpl.java | 4 +- .../core/client/impl/SyncingTabletLocator.java | 115 .../core/client/impl/TabletLocator.java | 12 + .../impl/TabletServerBatchReaderIterator.java | 4 +- .../client/impl/TabletServerBatchWriter.java| 9 +- .../core/client/impl/TimeoutTabletLocator.java | 48 +--- .../test/BatchWriterInTabletServerIT.java | 126 + .../accumulo/test/BatchWriterIterator.java | 49 ++-- .../apache/accumulo/test/SerializationUtil.java | 261 --- .../accumulo/test/util/SerializationUtil.java | 257 ++ .../test/BatchWriterInTabletServerIT.java | 118 - 11 files changed, 557 insertions(+), 446 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aac9c0f/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java index 00140e9..1c1aaf3 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java @@ -113,7 +113,7 @@ class ConditionalWriterImpl implements ConditionalWriter { private Map cache = Collections.synchronizedMap(new LRUMap(1000)); private final ClientContext context; private TabletLocator locator; - private String tableId; + private final String tableId; private long timeout; private final Durability durability; private final String classLoaderContext; @@ -388,7 +388,7 @@ class ConditionalWriterImpl implements ConditionalWriter { this.auths = config.getAuthorizations(); this.ve = new VisibilityEvaluator(config.getAuthorizations()); this.threadPool = new ScheduledThreadPoolExecutor(config.getMaxWriteThreads(), new NamingThreadFactory(this.getClass().getSimpleName())); -this.locator = TabletLocator.getLocator(context, tableId); +this.locator = new SyncingTabletLocator(context, tableId); this.serverQueues = new HashMap (); this.tableId = tableId; this.timeout = config.getTimeout(TimeUnit.MILLISECONDS); http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aac9c0f/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java new file mode 100644 index 000..6e7e072 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/SyncingTabletLocator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding
[1/2] accumulo git commit: ACCUMULO-4229 Add breaking test to reproduce sync problem
Repository: accumulo Updated Branches: refs/heads/1.8 c044127a9 -> 9aac9c0f0 ACCUMULO-4229 Add breaking test to reproduce sync problem Add two tests in BatchWriterInTabletServerIT. The first passes and establishes a baseline. The second fails, proving the issue. I added an extra iterator and a utility class for the IT. These are in the test package, so they do not get packaged into normal builds. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5275f335 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5275f335 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5275f335 Branch: refs/heads/1.8 Commit: 5275f335cf6ba22a017edaccf64e096f7d59ceb2 Parents: c044127 Author: Dylan HutchisonAuthored: Sat Apr 23 01:02:01 2016 -0700 Committer: Dylan Hutchison Committed: Thu May 5 14:14:40 2016 -0700 -- .../accumulo/test/BatchWriterIterator.java | 256 ++ .../apache/accumulo/test/SerializationUtil.java | 261 +++ .../test/BatchWriterInTabletServerIT.java | 118 + 3 files changed, 635 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/5275f335/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java b/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java new file mode 100644 index 000..7d44729 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/BatchWriterIterator.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.TimedOutException; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.impl.TabletLocator; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iterators.WrappingIterator; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; + +/** + * Iterator that opens a BatchWriter and writes to another table. + * + * For each entry passed to this iterator, this writes a certain number of entries with the same key to another table and passes the entry downstream of this + * iterator with its value replaced by either "{@value SUCCESS_STRING}" or a description of what failed. Success counts as all entries writing to the result + * table within a timeout period. Failure counts as one of the entries taking longer than the timeout period. + * + * Configure this iterator by calling the static {@link #iteratorSetting} method. + */ +public class BatchWriterIterator extends WrappingIterator { + private static final Logger log =
accumulo git commit: ACCUMULO-4299 Switch to Java 8
Repository: accumulo Updated Branches: refs/heads/master c044127a9 -> a40733374 ACCUMULO-4299 Switch to Java 8 Bump version to 2.0.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4073337 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4073337 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4073337 Branch: refs/heads/master Commit: a40733374075c47d7cb403b3fdab00fb19639acf Parents: c044127 Author: Christopher TubbsAuthored: Thu May 5 14:26:27 2016 -0400 Committer: Christopher Tubbs Committed: Thu May 5 14:26:27 2016 -0400 -- .travis.yml | 5 - assemble/pom.xml | 2 +- core/pom.xml | 2 +- docs/pom.xml | 2 +- examples/simple/pom.xml | 2 +- fate/pom.xml | 2 +- iterator-test-harness/pom.xml | 2 +- maven-plugin/pom.xml | 2 +- minicluster/pom.xml | 2 +- pom.xml | 31 --- proxy/pom.xml | 2 +- server/base/pom.xml | 2 +- server/gc/pom.xml | 2 +- server/master/pom.xml | 2 +- server/monitor/pom.xml| 2 +- server/native/pom.xml | 2 +- server/tracer/pom.xml | 2 +- server/tserver/pom.xml| 2 +- shell/pom.xml | 2 +- start/pom.xml | 2 +- test/pom.xml | 2 +- trace/pom.xml | 2 +- 22 files changed, 24 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 81672e5..c650fe4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. language: java -addons: - hostname: travis-accumulo-host notifications: irc: channels: @@ -28,11 +26,8 @@ cache: install: true jdk: - oraclejdk8 - - oraclejdk7 - - openjdk7 # clear any cache of accumulo artifacts, just in case; use https for central before_script: - - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname)/" /etc/hosts | sudo tee /etc/hosts - rm -rf $HOME/.m2/repository/org/apache/accumulo - echo "secure-centralhttps://repo.maven.apache.org/maven2central" > $HOME/settings-custom.xml script: mvn --settings $HOME/settings-custom.xml clean verify -DskipITs http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/assemble/pom.xml -- diff --git a/assemble/pom.xml b/assemble/pom.xml index a49e7b9..be8bfa6 100644 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@ -20,7 +20,7 @@ org.apache.accumulo accumulo-project -1.8.0-SNAPSHOT +2.0.0-SNAPSHOT accumulo pom http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index b412933..b00a65f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ org.apache.accumulo accumulo-project -1.8.0-SNAPSHOT +2.0.0-SNAPSHOT accumulo-core Apache Accumulo Core http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/docs/pom.xml -- diff --git a/docs/pom.xml b/docs/pom.xml index 6445efb..aa1a8ec 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -20,7 +20,7 @@ org.apache.accumulo accumulo-project -1.8.0-SNAPSHOT +2.0.0-SNAPSHOT accumulo-docs pom http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/examples/simple/pom.xml -- diff --git a/examples/simple/pom.xml b/examples/simple/pom.xml index b5ecbd9..0debf5c 100644 --- a/examples/simple/pom.xml +++ b/examples/simple/pom.xml @@ -20,7 +20,7 @@ org.apache.accumulo accumulo-project -1.8.0-SNAPSHOT +2.0.0-SNAPSHOT ../../pom.xml accumulo-examples-simple http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/fate/pom.xml -- diff --git a/fate/pom.xml b/fate/pom.xml index 059e4f1..bc665dc 100644 --- a/fate/pom.xml +++ b/fate/pom.xml @@ -20,7 +20,7 @@ org.apache.accumulo accumulo-project -1.8.0-SNAPSHOT +2.0.0-SNAPSHOT accumulo-fate Apache Accumulo Fate http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4073337/iterator-test-harness/pom.xml
[1/3] accumulo git commit: ACCUMULO-4298: Add env variables to make NUMA configurable
Repository: accumulo Updated Branches: refs/heads/1.8 ce7c1fa5d -> c044127a9 ACCUMULO-4298: Add env variables to make NUMA configurable Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f71f2933 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f71f2933 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f71f2933 Branch: refs/heads/1.8 Commit: f71f29338d6b50c57a85dfc720cd7dd9b6fa0bfd Parents: 397de58 Author: Dave MarionAuthored: Thu May 5 13:32:04 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:32:04 2016 -0400 -- assemble/bin/config.sh | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f71f2933/assemble/bin/config.sh -- diff --git a/assemble/bin/config.sh b/assemble/bin/config.sh index 9bee91b..89a5e48 100755 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@ -42,6 +42,10 @@ # Values set by script if certain files exist # ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins # ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers +# +# NUMA related environment variables +# ACCUMULO_ENABLE_NUMACTL (Default: true) Use numactl if present on the system +# ACCUMULO_NUMACTL_OPTIONS (Default: --interleave=all) options to the numactl command if [ -z "${ACCUMULO_HOME}" ] ; then # Start: Resolve Script Directory @@ -149,11 +153,12 @@ if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then fi fi +ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} +ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} NUMA=`which numactl 2>/dev/null` NUMACTL_EXISTS="$?" -NUMACTL_ARGS="--interleave=all" -if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" +if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
[3/3] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c044127a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c044127a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c044127a Branch: refs/heads/1.8 Commit: c044127a98a47707d10f55a2c4b28d86d5642a2d Parents: ce7c1fa 160e2c7 Author: Dave MarionAuthored: Thu May 5 13:41:38 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:41:38 2016 -0400 -- assemble/bin/config.sh | 15 ++- 1 file changed, 10 insertions(+), 5 deletions(-) --
[2/3] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/160e2c79 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/160e2c79 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/160e2c79 Branch: refs/heads/1.8 Commit: 160e2c796a516736a71e67719b3464b79d46c8f6 Parents: 523e790 f71f293 Author: Dave MarionAuthored: Thu May 5 13:41:27 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:41:27 2016 -0400 -- assemble/bin/config.sh | 15 ++- 1 file changed, 10 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/160e2c79/assemble/bin/config.sh -- diff --cc assemble/bin/config.sh index 9dfd164,89a5e48..2299a12 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@@ -104,11 -119,46 +108,12 @@@ the fi export HADOOP_PREFIX - NUMA=$(which numactl 2>/dev/null) - NUMACTL_EXISTS=$? - NUMACTL_ARGS='--interleave=all' - if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" -unset MASTER1 -if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then - MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) -fi - -if [ -z "${MONITOR}" ] ; then - MONITOR=$MASTER1 - if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then - MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) - fi - if [ -z "${MONITOR}" ] ; then -echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - fi -fi -if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then -echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - else -echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers" - fi - -fi - -if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then -echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - else -echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc" - fi -fi - + ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} + ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + NUMA=`which numactl 2>/dev/null` + NUMACTL_EXISTS="$?" + if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
[1/2] accumulo git commit: ACCUMULO-4298: Add env variables to make NUMA configurable
Repository: accumulo Updated Branches: refs/heads/1.7 523e7905f -> 160e2c796 ACCUMULO-4298: Add env variables to make NUMA configurable Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f71f2933 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f71f2933 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f71f2933 Branch: refs/heads/1.7 Commit: f71f29338d6b50c57a85dfc720cd7dd9b6fa0bfd Parents: 397de58 Author: Dave MarionAuthored: Thu May 5 13:32:04 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:32:04 2016 -0400 -- assemble/bin/config.sh | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f71f2933/assemble/bin/config.sh -- diff --git a/assemble/bin/config.sh b/assemble/bin/config.sh index 9bee91b..89a5e48 100755 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@ -42,6 +42,10 @@ # Values set by script if certain files exist # ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins # ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers +# +# NUMA related environment variables +# ACCUMULO_ENABLE_NUMACTL (Default: true) Use numactl if present on the system +# ACCUMULO_NUMACTL_OPTIONS (Default: --interleave=all) options to the numactl command if [ -z "${ACCUMULO_HOME}" ] ; then # Start: Resolve Script Directory @@ -149,11 +153,12 @@ if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then fi fi +ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} +ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} NUMA=`which numactl 2>/dev/null` NUMACTL_EXISTS="$?" -NUMACTL_ARGS="--interleave=all" -if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" +if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
[2/2] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/160e2c79 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/160e2c79 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/160e2c79 Branch: refs/heads/1.7 Commit: 160e2c796a516736a71e67719b3464b79d46c8f6 Parents: 523e790 f71f293 Author: Dave MarionAuthored: Thu May 5 13:41:27 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:41:27 2016 -0400 -- assemble/bin/config.sh | 15 ++- 1 file changed, 10 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/160e2c79/assemble/bin/config.sh -- diff --cc assemble/bin/config.sh index 9dfd164,89a5e48..2299a12 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@@ -104,11 -119,46 +108,12 @@@ the fi export HADOOP_PREFIX - NUMA=$(which numactl 2>/dev/null) - NUMACTL_EXISTS=$? - NUMACTL_ARGS='--interleave=all' - if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" -unset MASTER1 -if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then - MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) -fi - -if [ -z "${MONITOR}" ] ; then - MONITOR=$MASTER1 - if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then - MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) - fi - if [ -z "${MONITOR}" ] ; then -echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - fi -fi -if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then -echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - else -echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers" - fi - -fi - -if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then -echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - else -echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc" - fi -fi - + ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} + ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + NUMA=`which numactl 2>/dev/null` + NUMACTL_EXISTS="$?" + if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
[3/3] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c044127a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c044127a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c044127a Branch: refs/heads/master Commit: c044127a98a47707d10f55a2c4b28d86d5642a2d Parents: ce7c1fa 160e2c7 Author: Dave MarionAuthored: Thu May 5 13:41:38 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:41:38 2016 -0400 -- assemble/bin/config.sh | 15 ++- 1 file changed, 10 insertions(+), 5 deletions(-) --
[1/3] accumulo git commit: ACCUMULO-4298: Add env variables to make NUMA configurable
Repository: accumulo Updated Branches: refs/heads/master ce7c1fa5d -> c044127a9 ACCUMULO-4298: Add env variables to make NUMA configurable Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f71f2933 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f71f2933 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f71f2933 Branch: refs/heads/master Commit: f71f29338d6b50c57a85dfc720cd7dd9b6fa0bfd Parents: 397de58 Author: Dave MarionAuthored: Thu May 5 13:32:04 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:32:04 2016 -0400 -- assemble/bin/config.sh | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f71f2933/assemble/bin/config.sh -- diff --git a/assemble/bin/config.sh b/assemble/bin/config.sh index 9bee91b..89a5e48 100755 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@ -42,6 +42,10 @@ # Values set by script if certain files exist # ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins # ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers +# +# NUMA related environment variables +# ACCUMULO_ENABLE_NUMACTL (Default: true) Use numactl if present on the system +# ACCUMULO_NUMACTL_OPTIONS (Default: --interleave=all) options to the numactl command if [ -z "${ACCUMULO_HOME}" ] ; then # Start: Resolve Script Directory @@ -149,11 +153,12 @@ if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then fi fi +ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} +ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} NUMA=`which numactl 2>/dev/null` NUMACTL_EXISTS="$?" -NUMACTL_ARGS="--interleave=all" -if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" +if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
[2/3] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/160e2c79 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/160e2c79 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/160e2c79 Branch: refs/heads/master Commit: 160e2c796a516736a71e67719b3464b79d46c8f6 Parents: 523e790 f71f293 Author: Dave MarionAuthored: Thu May 5 13:41:27 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:41:27 2016 -0400 -- assemble/bin/config.sh | 15 ++- 1 file changed, 10 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/160e2c79/assemble/bin/config.sh -- diff --cc assemble/bin/config.sh index 9dfd164,89a5e48..2299a12 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@@ -104,11 -119,46 +108,12 @@@ the fi export HADOOP_PREFIX - NUMA=$(which numactl 2>/dev/null) - NUMACTL_EXISTS=$? - NUMACTL_ARGS='--interleave=all' - if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" -unset MASTER1 -if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then - MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) -fi - -if [ -z "${MONITOR}" ] ; then - MONITOR=$MASTER1 - if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then - MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) - fi - if [ -z "${MONITOR}" ] ; then -echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - fi -fi -if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then -echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - else -echo "$MASTER1" > "$ACCUMULO_CONF_DIR/tracers" - fi - -fi - -if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then -echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." -exit 1 - else -echo "$MASTER1" > "$ACCUMULO_CONF_DIR/gc" - fi -fi - + ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} + ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} + NUMA=`which numactl 2>/dev/null` + NUMACTL_EXISTS="$?" + if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
accumulo git commit: ACCUMULO-4298: Add env variables to make NUMA configurable
Repository: accumulo Updated Branches: refs/heads/1.6 397de589e -> f71f29338 ACCUMULO-4298: Add env variables to make NUMA configurable Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f71f2933 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f71f2933 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f71f2933 Branch: refs/heads/1.6 Commit: f71f29338d6b50c57a85dfc720cd7dd9b6fa0bfd Parents: 397de58 Author: Dave MarionAuthored: Thu May 5 13:32:04 2016 -0400 Committer: Dave Marion Committed: Thu May 5 13:32:04 2016 -0400 -- assemble/bin/config.sh | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f71f2933/assemble/bin/config.sh -- diff --git a/assemble/bin/config.sh b/assemble/bin/config.sh index 9bee91b..89a5e48 100755 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@ -42,6 +42,10 @@ # Values set by script if certain files exist # ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins # ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers +# +# NUMA related environment variables +# ACCUMULO_ENABLE_NUMACTL (Default: true) Use numactl if present on the system +# ACCUMULO_NUMACTL_OPTIONS (Default: --interleave=all) options to the numactl command if [ -z "${ACCUMULO_HOME}" ] ; then # Start: Resolve Script Directory @@ -149,11 +153,12 @@ if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then fi fi +ACCUMULO_ENABLE_NUMACTL=${ACCUMULO_ENABLE_NUMACTL:-"true"} +ACCUMULO_NUMACTL_OPTIONS=${ACCUMULO_NUMACTL_OPTIONS:-"--interleave=all"} NUMA=`which numactl 2>/dev/null` NUMACTL_EXISTS="$?" -NUMACTL_ARGS="--interleave=all" -if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then - export NUMA_CMD="${NUMA} ${NUMACTL_ARGS}" +if [[ ( ${NUMACTL_EXISTS} -eq 0 ) && ( ${ACCUMULO_ENABLE_NUMACTL} == "true" ) ]] ; then + export NUMA_CMD="${NUMA} ${ACCUMULO_NUMACTL_OPTIONS}" else export NUMA_CMD="" fi
[accumulo] Git Push Summary
Repository: accumulo Updated Branches: refs/heads/1.8 [created] ce7c1fa5d