ACCUMULO-3638 merge master branch
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4635de86 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4635de86 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4635de86 Branch: refs/heads/master Commit: 4635de8671a62f0a412da775db05519f6831daaa Parents: 98c3cef 9f108c0 Author: Eric C. Newton <eric.new...@gmail.com> Authored: Tue Mar 10 16:16:53 2015 -0400 Committer: Eric C. Newton <eric.new...@gmail.com> Committed: Tue Mar 10 16:16:53 2015 -0400 ---------------------------------------------------------------------- core/pom.xml | 7 -- core/src/main/findbugs/exclude-filter.xml | 67 +++++++++++++----- .../client/admin/CompactionStrategyConfig.java | 5 ++ .../client/impl/MultiTableBatchWriterImpl.java | 13 ++-- .../core/client/impl/TabletLocatorImpl.java | 41 +++++++---- .../client/impl/TabletServerBatchWriter.java | 45 ++++++++---- .../core/client/mock/MockNamespace.java | 2 +- .../client/security/tokens/DelegationToken.java | 16 ++--- .../client/security/tokens/PasswordToken.java | 13 ++-- .../apache/accumulo/core/conf/PropertyType.java | 2 +- .../org/apache/accumulo/core/data/Value.java | 1 + .../apache/accumulo/core/volume/VolumeImpl.java | 6 ++ .../accumulo/core/cli/TestClientOpts.java | 2 +- .../core/file/rfile/RFileMetricsTest.java | 73 +++++++++----------- .../iterators/user/ColumnSliceFilterTest.java | 2 +- .../core/iterators/user/RowFilterTest.java | 2 +- .../simple/src/main/findbugs/exclude-filter.xml | 18 +++++ .../examples/simple/mapreduce/RowHash.java | 2 +- fate/src/main/findbugs/exclude-filter.xml | 18 +++++ .../accumulo/fate/zookeeper/ZooReader.java | 14 ++-- .../zookeeper/DistributedReadWriteLockTest.java | 17 +++-- .../src/main/findbugs/exclude-filter.xml | 18 +++++ .../src/main/findbugs/exclude-filter.xml | 30 ++++++++ pom.xml | 3 +- proxy/pom.xml | 13 ---- proxy/src/main/findbugs/exclude-filter.xml | 7 +- .../base/src/main/findbugs/exclude-filter.xml | 26 +++++++ .../server/master/balancer/GroupBalancer.java | 6 ++ .../master/state/TabletLocationState.java | 3 +- .../server/metrics/MetricsConfiguration.java | 7 +- .../TCredentialsUpdatingInvocationHandler.java | 5 +- .../server/security/SecurityOperation.java | 7 -- .../security/delegation/AuthenticationKey.java | 11 +-- .../security/handler/KerberosAuthenticator.java | 4 +- .../server/watcher/Log4jConfiguration.java | 5 +- .../accumulo/server/ServerConstantsTest.java | 12 +++- .../AuthenticationTokenKeyManagerTest.java | 34 +++++---- server/gc/src/main/findbugs/exclude-filter.xml | 18 +++++ .../gc/GarbageCollectWriteAheadLogs.java | 2 +- .../master/src/main/findbugs/exclude-filter.xml | 23 ++++++ .../java/org/apache/accumulo/master/Master.java | 24 ++++--- .../RemoveCompleteReplicationRecords.java | 3 + .../src/main/findbugs/exclude-filter.xml | 18 +++++ .../monitor/servlets/OperationServlet.java | 53 +++++++++----- server/tracer/pom.xml | 13 ---- .../tracer/src/main/findbugs/exclude-filter.xml | 7 +- .../src/main/findbugs/exclude-filter.xml | 24 +++++++ .../accumulo/server/logger/LogFileKey.java | 25 ------- .../accumulo/server/logger/LogFileValue.java | 25 ------- .../accumulo/tserver/log/LocalWALRecovery.java | 4 +- .../replication/AccumuloReplicaSystem.java | 19 ++++- .../apache/accumulo/tserver/scan/ScanTask.java | 30 ++++++-- .../apache/accumulo/tserver/tablet/Tablet.java | 14 ++-- .../tserver/tablet/TabletCommitter.java | 3 + shell/src/main/findbugs/exclude-filter.xml | 18 +++++ .../apache/accumulo/shell/ShellConfigTest.java | 29 ++++++-- .../org/apache/accumulo/shell/ShellTest.java | 11 ++- .../shell/commands/FormatterCommandTest.java | 6 +- start/src/main/findbugs/exclude-filter.xml | 18 +++++ .../start/classloader/vfs/MiniDFSUtil.java | 4 ++ test/src/main/findbugs/exclude-filter.xml | 30 ++++++++ .../server/security/SystemCredentialsIT.java | 2 + .../test/MasterRepairsDualAssignmentIT.java | 2 +- .../accumulo/test/MultiTableRecoveryIT.java | 3 +- .../java/org/apache/accumulo/test/UsersIT.java | 60 ++++++++++++++++ .../apache/accumulo/test/functional/BulkIT.java | 13 ++-- .../test/functional/ConfigurableMacIT.java | 4 +- .../accumulo/test/functional/MonitorSslIT.java | 40 ++++++----- .../accumulo/test/functional/PermissionsIT.java | 43 ++++++++---- .../apache/accumulo/test/functional/SslIT.java | 5 +- trace/src/main/findbugs/exclude-filter.xml | 26 +++++++ 71 files changed, 810 insertions(+), 366 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java index 3ece3c9,b24b562..ebad2c8 --- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java @@@ -33,8 -32,6 +32,8 @@@ import org.apache.hadoop.io.Text */ public class TabletLocationState { - private static final Logger log = Logger.getLogger(TabletLocationState.class); ++ // private static final Logger log = Logger.getLogger(TabletLocationState.class); + static public class BadLocationStateException extends Exception { private static final long serialVersionUID = 1L; private Text metadataTableEntry; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java ---------------------------------------------------------------------- diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index 444789b,35c60d6..f44a9d1 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@@ -214,39 -268,76 +214,39 @@@ public class GarbageCollectWriteAheadLo } } } - - return 0; + return status.currentLog.deleted; } - /** - * Converts a list of paths to their corresponding strings. - * - * @param paths - * list of paths - * @return string forms of paths - */ - static List<String> paths2strings(List<Path> paths) { - List<String> result = new ArrayList<String>(paths.size()); - for (Path path : paths) - result.add(path.toString()); - return result; - } + private long removeMetadataEntries(Map<TServerInstance, Set<Path> > candidates, GCStatus status, Set<TServerInstance> liveServers) throws IOException, KeeperException, + InterruptedException { - /** - * Reverses the given mapping of file paths to servers. The returned map provides a list of file paths for each server. Any path whose name is not in the - * mapping of file names to paths is skipped. - * - * @param fileToServerMap - * map of file paths to servers - * @param nameToFileMap - * map of file names to paths - * @return map of servers to lists of file paths - */ - static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) { - Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>(); - for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) { - if (!nameToFileMap.containsKey(fileServer.getKey().getName())) - continue; - ArrayList<Path> files = result.get(fileServer.getValue()); - if (files == null) { - files = new ArrayList<Path>(); - result.put(fileServer.getValue(), files); + // remove any entries if there's a log reference, or a tablet is still assigned to the dead server + + Map<Path, TServerInstance> walToDeadServer = new HashMap<>(); + for (Entry<TServerInstance,Set<Path>> entry : candidates.entrySet()) { + for (Path file : entry.getValue()) { + walToDeadServer.put(file, entry.getKey()); } - files.add(fileServer.getKey()); } - return result; - } - - protected int removeMetadataEntries(Map<String,Path> nameToFileMap, Map<String,Path> sortedWALogs, GCStatus status) throws IOException, KeeperException, - InterruptedException { - int count = 0; - Iterator<LogEntry> iterator = MetadataTableUtil.getLogEntries(context); - - // For each WAL reference in the metadata table - while (iterator.hasNext()) { - // Each metadata reference has at least one WAL file - for (String entry : iterator.next().logSet) { - // old style WALs will have the IP:Port of their logger and new style will either be a Path either absolute or relative, in all cases - // the last "/" will mark a UUID file name. - String uuid = entry.substring(entry.lastIndexOf("/") + 1); - if (!isUUID(uuid)) { - // fully expect this to be a uuid, if its not then something is wrong and walog GC should not proceed! - throw new IllegalArgumentException("Expected uuid, but got " + uuid + " from " + entry); - } - - Path pathFromNN = nameToFileMap.remove(uuid); - if (pathFromNN != null) { - status.currentLog.inUse++; - sortedWALogs.remove(uuid); + long count = 0; + RootTabletStateStore root = new RootTabletStateStore(context); + MetaDataStateStore meta = new MetaDataStateStore(context); + Iterator<TabletLocationState> states = Iterators.concat(root.iterator(), meta.iterator()); + while (states.hasNext()) { + count++; + TabletLocationState state = states.next(); + if (state.getState(liveServers) == TabletState.ASSIGNED_TO_DEAD_SERVER) { + candidates.remove(state.current); + } + for (Collection<String> wals : state.walogs) { + for (String wal : wals) { - TServerInstance dead = walToDeadServer.get(wal); ++ TServerInstance dead = walToDeadServer.get(new Path(wal)); + if (dead != null) { + candidates.get(dead).remove(wal); + } } - - count++; } } - return count; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/findbugs/exclude-filter.xml ---------------------------------------------------------------------- diff --cc server/tserver/src/main/findbugs/exclude-filter.xml index 0000000,aec413a..45f6a78 mode 000000,100644..100644 --- a/server/tserver/src/main/findbugs/exclude-filter.xml +++ b/server/tserver/src/main/findbugs/exclude-filter.xml @@@ -1,0 -1,24 +1,24 @@@ + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <FindBugsFilter> + <Match> + <!-- locking is confusing, but probably correct --> + <Class name="org.apache.accumulo.tserver.tablet.Tablet" /> - <Method name="beginUpdatingLogsUsed" params="org.apache.accumulo.tserver.InMemoryMap,java.util.Collection,boolean" returns="boolean" /> ++ <Method name="beginUpdatingLogsUsed" params="org.apache.accumulo.tserver.InMemoryMap,org.apache.accumulo.tserver.log.DfsLogger,boolean" returns="boolean" /> + <Bug code="UL" pattern="UL_UNRELEASED_LOCK" /> + </Match> + </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java ---------------------------------------------------------------------- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 0c1edfa,95fe24a..e964719 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@@ -2442,8 -2441,9 +2440,9 @@@ public class Tablet implements TabletCo return currentLogs.size(); } + /* don't release the lock if this method returns true for success; instead, the caller should clean up by calling finishUpdatingLogsUsed() */ @Override - public boolean beginUpdatingLogsUsed(InMemoryMap memTable, Collection<DfsLogger> more, boolean mincFinish) { + public boolean beginUpdatingLogsUsed(InMemoryMap memTable, DfsLogger more, boolean mincFinish) { boolean releaseLock = true; http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java ---------------------------------------------------------------------- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java index 39bde5c,4bc05a6..91a955e --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java @@@ -34,7 -35,10 +34,10 @@@ public interface TabletCommitter void commit(CommitSession commitSession, List<Mutation> mutations); + /** + * If this method returns true, the caller must call {@link #finishUpdatingLogsUsed()} to clean up + */ - boolean beginUpdatingLogsUsed(InMemoryMap memTable, Collection<DfsLogger> copy, boolean mincFinish); + boolean beginUpdatingLogsUsed(InMemoryMap memTable, DfsLogger copy, boolean mincFinish); void finishUpdatingLogsUsed();