ACCUMULO-1085 merge master
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9be9576d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9be9576d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9be9576d Branch: refs/heads/metrics2 Commit: 9be9576d9c8a798c27d9ec570149a6af8cac59cf Parents: b2456f6 09a761b Author: Eric C. Newton <eric.new...@gmail.com> Authored: Wed Nov 26 13:28:08 2014 -0500 Committer: Eric C. Newton <eric.new...@gmail.com> Committed: Wed Nov 26 13:28:08 2014 -0500 ---------------------------------------------------------------------- TESTING | 113 ++++++ assemble/bin/bootstrap_config.sh | 69 +++- assemble/bin/start-all.sh | 4 +- assemble/conf/templates/accumulo-site.xml | 21 +- .../apache/accumulo/core/cli/ClientOpts.java | 2 +- .../accumulo/core/client/ZooKeeperInstance.java | 13 +- .../core/client/impl/BatchWriterImpl.java | 10 +- .../client/impl/ClientConfigurationHelper.java | 143 ------- .../core/client/impl/ClientContext.java | 229 +++++++++++ .../core/client/impl/ConditionalWriterImpl.java | 48 ++- .../core/client/impl/ConnectorImpl.java | 73 ++-- .../client/impl/InstanceOperationsImpl.java | 65 ++-- .../accumulo/core/client/impl/MasterClient.java | 45 ++- .../client/impl/MultiTableBatchWriterImpl.java | 20 +- .../client/impl/NamespaceOperationsImpl.java | 42 +-- .../core/client/impl/ReplicationClient.java | 52 ++- .../client/impl/ReplicationOperationsImpl.java | 39 +- .../core/client/impl/RootTabletLocator.java | 29 +- .../accumulo/core/client/impl/ScannerImpl.java | 16 +- .../core/client/impl/ScannerIterator.java | 20 +- .../client/impl/SecurityOperationsImpl.java | 54 ++- .../accumulo/core/client/impl/ServerClient.java | 37 +- .../core/client/impl/TableOperationsImpl.java | 169 ++++----- .../core/client/impl/TabletLocator.java | 22 +- .../core/client/impl/TabletLocatorImpl.java | 61 ++- .../client/impl/TabletServerBatchDeleter.java | 16 +- .../client/impl/TabletServerBatchReader.java | 16 +- .../impl/TabletServerBatchReaderIterator.java | 45 +-- .../client/impl/TabletServerBatchWriter.java | 45 +-- .../core/client/impl/ThriftScanner.java | 52 ++- .../core/client/impl/ThriftTransportKey.java | 5 +- .../core/client/impl/ThriftTransportPool.java | 14 +- .../core/client/impl/TimeoutTabletLocator.java | 18 +- .../accumulo/core/client/impl/Writer.java | 38 +- .../client/mock/impl/MockTabletLocator.java | 11 +- .../core/metadata/MetadataLocationObtainer.java | 32 +- .../core/metadata/MetadataServicer.java | 18 +- .../core/metadata/ServicerForMetadataTable.java | 7 +- .../core/metadata/ServicerForRootTable.java | 8 +- .../core/metadata/ServicerForUserTables.java | 7 +- .../core/metadata/TableMetadataServicer.java | 13 +- .../AccumuloReplicationReplayer.java | 11 +- .../apache/accumulo/core/util/ThriftUtil.java | 42 +-- .../impl/ClientConfigurationHelperTest.java | 110 ------ .../core/client/impl/ClientContextTest.java | 109 ++++++ .../core/client/impl/RootTabletLocatorTest.java | 13 +- .../core/client/impl/ScannerImplTest.java | 7 +- .../client/impl/TableOperationsImplTest.java | 6 +- .../core/client/impl/TabletLocatorImplTest.java | 236 ++++++------ .../core/metadata/MetadataServicerTest.java | 15 +- .../ReplicationOperationsImplTest.java | 14 +- docs/src/main/asciidoc/chapters/clients.txt | 2 +- .../examples/simple/combiner/StatsCombiner.java | 2 +- .../core/client/mapred/AbstractInputFormat.java | 9 +- .../client/mapreduce/AbstractInputFormat.java | 9 +- .../mapreduce/lib/impl/InputConfigurator.java | 10 +- minicluster/pom.xml | 8 + .../accumulo/cluster/AccumuloCluster.java | 46 ++- .../accumulo/cluster/AccumuloClusters.java | 43 --- .../apache/accumulo/cluster/AccumuloConfig.java | 156 -------- .../apache/accumulo/cluster/ClusterControl.java | 83 ++++ .../apache/accumulo/cluster/RemoteShell.java | 94 +++++ .../accumulo/cluster/RemoteShellOptions.java | 126 +++++++ .../apache/accumulo/cluster/package-info.java | 27 -- .../standalone/StandaloneAccumuloCluster.java | 147 ++++++++ .../standalone/StandaloneClusterControl.java | 351 +++++++++++++++++ .../minicluster/MiniAccumuloCluster.java | 3 +- .../apache/accumulo/minicluster/ServerType.java | 2 +- .../impl/MiniAccumuloClusterControl.java | 375 +++++++++++++++++++ .../impl/MiniAccumuloClusterImpl.java | 192 +++------- .../impl/MiniAccumuloConfigImpl.java | 37 +- .../accumulo/cluster/AccumuloClustersTest.java | 63 ---- .../MiniAccumuloClusterStartStopTest.java | 44 ++- .../minicluster/impl/CleanShutdownMacTest.java | 11 +- .../impl/MiniAccumuloClusterImplTest.java | 7 +- pom.xml | 14 +- .../accumulo/server/AccumuloServerContext.java | 67 ++++ .../accumulo/server/client/BulkImporter.java | 97 ++--- .../server/client/ClientServiceHandler.java | 26 +- .../accumulo/server/client/HdfsZooInstance.java | 3 +- .../server/conf/ServerConfigurationFactory.java | 7 +- .../server/conf/TableConfiguration.java | 13 +- .../apache/accumulo/server/fs/VolumeUtil.java | 18 +- .../apache/accumulo/server/init/Initialize.java | 11 +- .../accumulo/server/master/LiveTServerSet.java | 70 ++-- .../master/balancer/TableLoadBalancer.java | 3 +- .../server/master/balancer/TabletBalancer.java | 13 +- .../server/master/state/MetaDataStateStore.java | 30 +- .../master/state/MetaDataTableScanner.java | 26 +- .../master/state/RootTabletStateStore.java | 14 +- .../server/master/state/TabletStateStore.java | 14 +- .../accumulo/server/problems/ProblemReport.java | 14 +- .../problems/ProblemReportingIterator.java | 12 +- .../server/problems/ProblemReports.java | 27 +- .../server/replication/ReplicationUtil.java | 65 ++-- .../security/AuditedSecurityOperation.java | 28 +- .../server/security/SecurityOperation.java | 31 +- .../server/security/SystemCredentials.java | 16 - .../accumulo/server/util/AccumuloStatus.java | 3 - .../org/apache/accumulo/server/util/Admin.java | 72 ++-- .../server/util/FindOfflineTablets.java | 23 +- .../accumulo/server/util/ListVolumesUsed.java | 16 +- .../server/util/MasterMetadataUtil.java | 43 ++- .../accumulo/server/util/MetadataTableUtil.java | 176 ++++----- .../accumulo/server/util/RandomizeVolumes.java | 7 +- .../util/RemoveEntriesForMissingFiles.java | 25 +- .../server/util/ReplicationTableUtil.java | 22 +- .../accumulo/server/util/TServerUtils.java | 18 +- .../server/util/VerifyTabletAssignments.java | 31 +- .../server/client/BulkImporterTest.java | 29 +- .../server/conf/TableConfigurationTest.java | 13 +- .../master/balancer/TableLoadBalancerTest.java | 28 +- .../problems/ProblemReportingIteratorTest.java | 6 +- .../server/security/SystemCredentialsTest.java | 17 +- .../accumulo/server/util/FileUtilTest.java | 66 ++-- .../server/util/ReplicationTableUtilTest.java | 9 +- .../gc/GarbageCollectWriteAheadLogs.java | 40 +- .../accumulo/gc/SimpleGarbageCollector.java | 114 ++---- .../CloseWriteAheadLogReferences.java | 46 +-- .../gc/GarbageCollectWriteAheadLogsTest.java | 51 ++- .../accumulo/gc/SimpleGarbageCollectorTest.java | 18 +- .../CloseWriteAheadLogReferencesTest.java | 38 +- .../java/org/apache/accumulo/master/Master.java | 146 ++++---- .../master/MasterClientServiceHandler.java | 46 +-- .../accumulo/master/TabletGroupWatcher.java | 9 +- .../master/metrics/ReplicationMetrics.java | 78 +--- .../MasterReplicationCoordinator.java | 5 +- .../master/replication/ReplicationDriver.java | 2 +- .../accumulo/master/replication/WorkDriver.java | 6 +- .../accumulo/master/replication/WorkMaker.java | 11 +- .../accumulo/master/tableOps/BulkImport.java | 22 +- .../master/tableOps/CancelCompactions.java | 11 +- .../master/tableOps/ChangeTableState.java | 9 +- .../accumulo/master/tableOps/CloneTable.java | 30 +- .../accumulo/master/tableOps/CompactRange.java | 26 +- .../master/tableOps/CreateNamespace.java | 5 +- .../accumulo/master/tableOps/CreateTable.java | 13 +- .../master/tableOps/DeleteNamespace.java | 3 +- .../accumulo/master/tableOps/DeleteTable.java | 20 +- .../accumulo/master/tableOps/ExportTable.java | 34 +- .../accumulo/master/tableOps/ImportTable.java | 16 +- .../accumulo/master/tableOps/RenameTable.java | 10 +- .../accumulo/master/tableOps/TableRangeOp.java | 13 +- .../apache/accumulo/master/TestMergeState.java | 10 +- .../MasterReplicationCoordinatorTest.java | 4 +- .../master/replication/WorkMakerTest.java | 17 +- .../accumulo/monitor/EmbeddedWebServer.java | 2 +- .../org/apache/accumulo/monitor/Monitor.java | 37 +- .../accumulo/monitor/servlets/BasicServlet.java | 13 +- .../accumulo/monitor/servlets/LogServlet.java | 2 +- .../monitor/servlets/MasterServlet.java | 9 +- .../monitor/servlets/OperationServlet.java | 10 +- .../monitor/servlets/ProblemServlet.java | 6 +- .../monitor/servlets/ReplicationServlet.java | 24 +- .../monitor/servlets/TServersServlet.java | 9 +- .../monitor/servlets/TablesServlet.java | 12 +- .../accumulo/monitor/servlets/VisServlet.java | 2 +- .../accumulo/monitor/servlets/XMLServlet.java | 3 +- .../accumulo/monitor/servlets/trace/Basic.java | 2 +- .../org/apache/accumulo/tracer/TraceDump.java | 24 +- .../org/apache/accumulo/tracer/TraceServer.java | 5 + .../apache/accumulo/tserver/FileManager.java | 44 +-- .../apache/accumulo/tserver/TabletServer.java | 91 +++-- .../tserver/TabletServerResourceManager.java | 22 +- .../tserver/log/TabletServerLogger.java | 11 +- .../replication/AccumuloReplicaSystem.java | 47 ++- .../BatchWriterReplicationReplayer.java | 14 +- .../replication/ReplicationServicerHandler.java | 35 +- .../tserver/replication/ReplicationWorker.java | 2 +- .../tserver/session/SessionManager.java | 8 +- .../accumulo/tserver/tablet/Compactor.java | 19 +- .../tserver/tablet/DatafileManager.java | 24 +- .../accumulo/tserver/tablet/MinorCompactor.java | 18 +- .../apache/accumulo/tserver/tablet/Tablet.java | 93 ++--- .../LargestFirstMemoryManagerTest.java | 28 +- .../BatchWriterReplicationReplayerTest.java | 41 +- .../java/org/apache/accumulo/shell/Shell.java | 17 +- .../shell/commands/DeleteTableCommand.java | 23 ++ .../accumulo/shell/commands/TableOperation.java | 16 +- .../shell/command/DropUserCommandTest.java | 83 ---- .../shell/command/FormatterCommandTest.java | 184 --------- .../shell/command/HistoryCommandTest.java | 90 ----- .../shell/commands/DeleteTableCommandTest.java | 42 +++ .../shell/commands/DropUserCommandTest.java | 83 ++++ .../shell/commands/FormatterCommandTest.java | 184 +++++++++ .../shell/commands/HistoryCommandTest.java | 90 +++++ .../apache/accumulo/test/GetMasterStats.java | 10 +- .../accumulo/test/TestMultiTableIngest.java | 20 +- .../apache/accumulo/test/WrongTabletTest.java | 18 +- .../continuous/ContinuousStatsCollector.java | 10 +- .../accumulo/test/functional/ZombieTServer.java | 14 +- .../metadata/MetadataBatchScanTest.java | 11 +- .../performance/scan/CollectTabletStats.java | 18 +- .../test/performance/thrift/NullTserver.java | 23 +- .../test/randomwalk/concurrent/Shutdown.java | 10 +- .../test/randomwalk/concurrent/StartAll.java | 10 +- .../randomwalk/security/WalkingSecurity.java | 9 +- .../accumulo/harness/AccumuloClusterIT.java | 220 +++++++++++ .../org/apache/accumulo/harness/AccumuloIT.java | 98 +++++ .../MiniClusterConfigurationCallback.java | 41 ++ .../accumulo/harness/MiniClusterHarness.java | 148 ++++++++ .../accumulo/harness/SharedMiniClusterIT.java | 87 +++++ .../conf/AccumuloClusterConfiguration.java | 33 ++ .../AccumuloClusterPropertyConfiguration.java | 180 +++++++++ .../conf/AccumuloMiniClusterConfiguration.java | 71 ++++ .../StandaloneAccumuloClusterConfiguration.java | 114 ++++++ .../server/security/SystemCredentialsIT.java | 2 +- .../test/AllowScansToBeInterruptedIT.java | 104 ----- .../test/ArbitraryTablePropertiesIT.java | 4 + .../apache/accumulo/test/AuditMessageIT.java | 44 ++- .../test/BadDeleteMarkersCreatedIT.java | 91 ++++- .../org/apache/accumulo/test/BatchWriterIT.java | 4 +- .../accumulo/test/BulkImportVolumeIT.java | 37 +- .../org/apache/accumulo/test/CleanWalIT.java | 71 +++- .../accumulo/test/ConditionalWriterIT.java | 70 ++-- .../test/ConfigurableMajorCompactionIT.java | 2 +- .../org/apache/accumulo/test/DeleteRowsIT.java | 47 --- .../org/apache/accumulo/test/ExistingMacIT.java | 5 +- .../apache/accumulo/test/ImportExportIT.java | 3 + .../accumulo/test/InterruptibleScannersIT.java | 105 ++++++ .../accumulo/test/KeyValueEqualityIT.java | 76 ++++ .../accumulo/test/KeyValueEqualityTest.java | 75 ---- .../test/MasterRepairsDualAssignmentIT.java | 7 +- .../accumulo/test/MetaConstraintRetryIT.java | 11 +- .../org/apache/accumulo/test/MetaSplitIT.java | 77 +++- .../accumulo/test/MultiTableBatchWriterIT.java | 40 +- .../org/apache/accumulo/test/NamespacesIT.java | 33 +- .../accumulo/test/NoMutationRecoveryIT.java | 39 +- .../test/RecoveryCompactionsAreFlushesIT.java | 22 +- .../org/apache/accumulo/test/ShellServerIT.java | 50 +-- .../accumulo/test/SplitCancelsMajCIT.java | 6 +- .../apache/accumulo/test/SplitRecoveryIT.java | 4 +- .../test/TableConfigurationUpdateIT.java | 8 +- .../apache/accumulo/test/TableOperationsIT.java | 31 +- .../org/apache/accumulo/test/TotalQueuedIT.java | 12 +- .../test/TracerRecoversAfterOfflineTableIT.java | 127 +++++++ .../java/org/apache/accumulo/test/VolumeIT.java | 17 +- .../accumulo/test/functional/AbstractMacIT.java | 83 +--- .../test/functional/AccumuloInputFormatIT.java | 6 +- .../accumulo/test/functional/AddSplitIT.java | 3 +- .../test/functional/BadIteratorMincIT.java | 3 +- .../functional/BalanceAfterCommsFailureIT.java | 6 +- .../BalanceInPresenceOfOfflineTableIT.java | 56 ++- .../test/functional/BatchScanSplitIT.java | 7 +- .../test/functional/BatchWriterFlushIT.java | 3 +- .../test/functional/BigRootTabletIT.java | 16 +- .../accumulo/test/functional/BinaryIT.java | 14 +- .../test/functional/BinaryStressIT.java | 78 +++- .../accumulo/test/functional/BloomFilterIT.java | 217 ++++++----- .../accumulo/test/functional/BulkFileIT.java | 20 +- .../apache/accumulo/test/functional/BulkIT.java | 44 ++- .../functional/BulkSplitOptimizationIT.java | 88 +++-- .../test/functional/ChaoticBalancerIT.java | 31 +- .../accumulo/test/functional/ClassLoaderIT.java | 18 +- .../accumulo/test/functional/CleanTmpIT.java | 55 ++- .../accumulo/test/functional/CleanUpIT.java | 76 +++- .../accumulo/test/functional/CloneTestIT.java | 16 +- .../accumulo/test/functional/CombinerIT.java | 3 +- .../accumulo/test/functional/CompactionIT.java | 85 ++++- .../accumulo/test/functional/ConcurrencyIT.java | 79 ++-- .../test/functional/ConfigurableMacIT.java | 12 +- .../accumulo/test/functional/ConstraintIT.java | 3 +- .../test/functional/CreateAndUseIT.java | 3 +- .../test/functional/CreateManyScannersIT.java | 3 +- .../accumulo/test/functional/CredentialsIT.java | 3 +- .../test/functional/DeleteEverythingIT.java | 76 ++-- .../accumulo/test/functional/DeleteIT.java | 37 +- .../accumulo/test/functional/DeleteRowsIT.java | 17 +- .../test/functional/DeleteRowsSplitIT.java | 3 +- .../functional/DeleteTableDuringSplitIT.java | 3 +- .../functional/DeletedTablesDontFlushIT.java | 7 +- .../test/functional/DynamicThreadPoolsIT.java | 53 ++- .../accumulo/test/functional/ExamplesIT.java | 251 ++++++++----- .../test/functional/FateStarvationIT.java | 3 +- .../accumulo/test/functional/LargeRowIT.java | 32 +- .../accumulo/test/functional/LogicalTimeIT.java | 31 +- .../test/functional/MasterAssignmentIT.java | 15 +- .../test/functional/MasterFailoverIT.java | 38 +- .../accumulo/test/functional/MaxOpenIT.java | 103 +++-- .../accumulo/test/functional/MergeIT.java | 3 +- .../accumulo/test/functional/MetadataIT.java | 6 +- .../test/functional/MetadataMaxFiles.java | 112 ------ .../test/functional/MetadataMaxFilesIT.java | 114 ++++++ .../accumulo/test/functional/PermissionsIT.java | 39 +- .../accumulo/test/functional/ReadWriteIT.java | 256 +++++++++---- .../functional/RecoveryWithEmptyRFileIT.java | 20 +- .../accumulo/test/functional/RenameIT.java | 3 +- .../accumulo/test/functional/RestartIT.java | 250 ++++++++++--- .../test/functional/RestartStressIT.java | 88 ++++- .../accumulo/test/functional/RowDeleteIT.java | 62 +-- .../test/functional/ScanIteratorIT.java | 3 +- .../accumulo/test/functional/ScanRangeIT.java | 3 +- .../test/functional/ScanSessionTimeOutIT.java | 80 ++-- .../accumulo/test/functional/ScannerIT.java | 7 +- .../test/functional/ServerSideErrorIT.java | 3 +- .../functional/SimpleBalancerFairnessIT.java | 8 +- .../accumulo/test/functional/SimpleMacIT.java | 16 +- .../test/functional/SparseColumnFamilyIT.java | 3 +- .../accumulo/test/functional/SplitIT.java | 140 +++++-- .../test/functional/SplitRecoveryIT.java | 76 ++-- .../apache/accumulo/test/functional/SslIT.java | 7 +- .../accumulo/test/functional/StartIT.java | 14 +- .../accumulo/test/functional/TableIT.java | 18 +- .../accumulo/test/functional/TabletIT.java | 7 +- .../accumulo/test/functional/TimeoutIT.java | 5 +- .../accumulo/test/functional/VisibilityIT.java | 3 +- .../test/functional/WriteAheadLogIT.java | 30 +- .../accumulo/test/functional/WriteLotsIT.java | 3 +- .../test/replication/CyclicReplicationIT.java | 9 +- ...bageCollectorCommunicatesWithTServersIT.java | 17 +- .../replication/MultiInstanceReplicationIT.java | 18 +- .../test/replication/ReplicationIT.java | 2 +- .../test/replication/StatusCombinerMacIT.java | 115 ++++++ .../test/replication/StatusCombinerMacTest.java | 114 ------ .../UnorderedWorkAssignerReplicationIT.java | 21 +- 315 files changed, 8353 insertions(+), 5087 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9be9576d/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index fd53756,5b49529..d4447ab --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@@ -238,10 -239,9 +238,10 @@@ import org.apache.thrift.server.TServer import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NoNodeException; -import com.google.common.net.HostAndPort; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD; - public class TabletServer implements Runnable { + public class TabletServer extends AccumuloServerContext implements Runnable { private static final Logger log = Logger.getLogger(TabletServer.class); private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000; private static final long RECENTLY_SPLIT_MILLIES = 60 * 1000; @@@ -2144,8 -2134,9 +2139,8 @@@ if (tablet.getNumEntriesInMemory() > 0 && !tablet.minorCompactNow(MinorCompactionReason.RECOVERY)) { throw new RuntimeException("Minor compaction after recovery fails for " + extent); } - Assignment assignment = new Assignment(extent, getTabletSession()); - TabletStateStore.setLocation(assignment); + TabletStateStore.setLocation(TabletServer.this, assignment); synchronized (openingTablets) { synchronized (onlineTablets) { @@@ -2162,9 -2153,7 +2157,9 @@@ if (e.getMessage() != null) log.warn(e.getMessage()); String table = extent.getTableId().toString(); - ProblemReports.getInstance().report(new ProblemReport(table, TABLET_LOAD, extent.getUUID().toString(), getClientAddressString(), e)); + ProblemReports.getInstance(TabletServer.this).report(new ProblemReport(table, TABLET_LOAD, extent.getUUID().toString(), getClientAddressString(), e)); + } finally { + releaseRecoveryMemory(extent); } if (!successful) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/9be9576d/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java ----------------------------------------------------------------------
