This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3aa65f8e072a2b5363fed30421fa64e63d14aeca
Merge: 703710e 8dfe443
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Tue Apr 20 13:49:08 2021 +0000
Merge branch 'main' into 1451-external-compactions-feature
Conflicts:
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
core/src/main/scripts/generate-thrift.sh
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
.github/workflows/maven-full-its.yaml | 19 +-
.github/workflows/maven-on-demand.yaml | 17 +-
.github/workflows/maven.yaml | 19 +-
.../core/client/sample/RowColumnSampler.java | 2 +-
.../accumulo/core/client/sample/RowSampler.java | 2 +-
.../client/summary/SummarizerConfiguration.java | 5 +-
.../accumulo/core/clientImpl/ManagerClient.java | 2 +-
.../core/clientImpl/NamespaceOperationsImpl.java | 2 +-
.../apache/accumulo/core/clientImpl/TableMap.java | 4 +-
.../core/clientImpl/TableOperationsImpl.java | 4 +-
.../accumulo/core/conf/AccumuloConfiguration.java | 2 +-
.../org/apache/accumulo/core/data/LoadPlan.java | 2 +-
.../core/metadata/schema/TabletMetadata.java | 17 +-
.../core/spi/scan/HintScanPrioritizer.java | 18 +-
.../core/spi/scan/SimpleScanDispatcher.java | 42 +-
.../accumulo/core/summary/SummarySerializer.java | 2 +-
.../accumulo/core/util/ConfigurationImpl.java | 12 +-
.../accumulo/core/util/LocalityGroupUtil.java | 35 +-
core/src/main/scripts/generate-thrift.sh | 4 +-
core/src/main/spotbugs/exclude-filter.xml | 1 +
.../core/clientImpl/thrift/ClientService.java | 0
.../core/clientImpl/thrift/ConfigurationType.java | 0
.../core/clientImpl/thrift/SecurityErrorCode.java | 0
.../core/clientImpl/thrift/TDiskUsage.java | 0
.../core/clientImpl/thrift/TableOperation.java | 0
.../thrift/TableOperationExceptionType.java | 0
.../thrift/ThriftNotActiveServiceException.java | 0
.../clientImpl/thrift/ThriftSecurityException.java | 0
.../thrift/ThriftTableOperationException.java | 0
.../core/clientImpl/thrift/ThriftTest.java | 0
.../core/dataImpl/thrift/InitialMultiScan.java | 0
.../accumulo/core/dataImpl/thrift/InitialScan.java | 0
.../accumulo/core/dataImpl/thrift/IterInfo.java | 0
.../accumulo/core/dataImpl/thrift/MapFileInfo.java | 0
.../core/dataImpl/thrift/MultiScanResult.java | 0
.../accumulo/core/dataImpl/thrift/ScanResult.java | 0
.../accumulo/core/dataImpl/thrift/TCMResult.java | 0
.../accumulo/core/dataImpl/thrift/TCMStatus.java | 0
.../accumulo/core/dataImpl/thrift/TColumn.java | 0
.../accumulo/core/dataImpl/thrift/TCondition.java | 0
.../core/dataImpl/thrift/TConditionalMutation.java | 0
.../core/dataImpl/thrift/TConditionalSession.java | 0
.../thrift/TConstraintViolationSummary.java | 0
.../apache/accumulo/core/dataImpl/thrift/TKey.java | 0
.../accumulo/core/dataImpl/thrift/TKeyExtent.java | 0
.../accumulo/core/dataImpl/thrift/TKeyValue.java | 0
.../accumulo/core/dataImpl/thrift/TMutation.java | 0
.../accumulo/core/dataImpl/thrift/TRange.java | 0
.../accumulo/core/dataImpl/thrift/TRowRange.java | 0
.../accumulo/core/dataImpl/thrift/TSummaries.java | 0
.../dataImpl/thrift/TSummarizerConfiguration.java | 0
.../accumulo/core/dataImpl/thrift/TSummary.java | 0
.../core/dataImpl/thrift/TSummaryRequest.java | 0
.../core/dataImpl/thrift/UpdateErrors.java | 0
.../accumulo/core/gc/thrift/GCMonitorService.java | 0
.../apache/accumulo/core/gc/thrift/GCStatus.java | 0
.../accumulo/core/gc/thrift/GcCycleStats.java | 0
.../accumulo/core/manager}/thrift/DeadServer.java | 2 +-
.../core/manager}/thrift/FateOperation.java | 2 +-
.../accumulo/core/manager}/thrift/FateService.java | 242 ++++----
.../core/manager}/thrift/ManagerClientService.java | 676 ++++++++++-----------
.../core/manager}/thrift/ManagerGoalState.java | 4 +-
.../core/manager}/thrift/ManagerMonitorInfo.java | 374 ++++++------
.../core/manager}/thrift/ManagerState.java | 4 +-
.../core/manager}/thrift/RecoveryException.java | 2 +-
.../core/manager}/thrift/TabletLoadState.java | 2 +-
.../accumulo/core/manager}/thrift/TabletSplit.java | 38 +-
.../core/master/thrift/BulkImportState.java | 0
.../core/master/thrift/BulkImportStatus.java | 0
.../accumulo/core/master/thrift/Compacting.java | 0
.../core/master/thrift/RecoveryStatus.java | 0
.../accumulo/core/master/thrift/TableInfo.java | 0
.../core/master/thrift/TabletServerStatus.java | 0
.../core/replication/thrift/KeyValues.java | 0
.../thrift/RemoteReplicationErrorCode.java | 0
.../thrift/RemoteReplicationException.java | 0
.../replication/thrift/ReplicationCoordinator.java | 0
.../thrift/ReplicationCoordinatorErrorCode.java | 0
.../thrift/ReplicationCoordinatorException.java | 0
.../replication/thrift/ReplicationServicer.java | 0
.../accumulo/core/replication/thrift/WalEdits.java | 0
.../securityImpl/thrift/TAuthenticationKey.java | 0
.../thrift/TAuthenticationTokenIdentifier.java | 0
.../core/securityImpl/thrift/TCredentials.java | 0
.../core/securityImpl/thrift/TDelegationToken.java | 0
.../thrift/TDelegationTokenConfig.java | 0
.../core/tabletserver/thrift/ActionStats.java | 0
.../core/tabletserver/thrift/ActiveCompaction.java | 0
.../core/tabletserver/thrift/ActiveScan.java | 0
.../core/tabletserver/thrift/CompactionReason.java | 0
.../core/tabletserver/thrift/CompactionType.java | 0
.../thrift/ConstraintViolationException.java | 0
.../core/tabletserver/thrift/IteratorConfig.java | 0
.../tabletserver/thrift/NoSuchScanIDException.java | 0
.../thrift/NotServingTabletException.java | 0
.../core/tabletserver/thrift/ScanState.java | 0
.../core/tabletserver/thrift/ScanType.java | 0
.../core/tabletserver/thrift/TDurability.java | 0
.../core/tabletserver/thrift/TIteratorSetting.java | 0
.../thrift/TSampleNotPresentException.java | 0
.../tabletserver/thrift/TSamplerConfiguration.java | 0
.../tabletserver/thrift/TUnloadTabletGoal.java | 0
.../tabletserver/thrift/TabletClientService.java | 0
.../core/tabletserver/thrift/TabletStats.java | 0
.../tabletserver/thrift/TooManyFilesException.java | 0
.../apache/accumulo/core/trace/thrift/TInfo.java | 0
core/src/main/thrift/manager.thrift | 269 +++-----
core/src/main/thrift/master.thrift | 93 +++
core/src/main/thrift/tabletserver.thrift | 3 +-
.../iterators/FirstEntryInRowIteratorTest.java | 4 +-
.../core/iterators/system/MultiIteratorTest.java | 6 +-
.../core/iterators/user/LargeRowFilterTest.java | 30 +-
.../BaseHostRegexTableLoadBalancerTest.java | 10 +-
.../standalone/StandaloneAccumuloCluster.java | 2 +-
.../standalone/StandaloneClusterControl.java | 2 +-
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 6 +-
.../standalone/StandaloneAccumuloClusterTest.java | 2 +-
.../MiniAccumuloClusterImplTest.java | 6 +-
pom.xml | 50 +-
.../accumulo/server/conf/TableConfiguration.java | 10 +-
.../apache/accumulo/server/init/Initialize.java | 2 +-
.../server/manager/state/CurrentState.java | 2 +-
.../server/manager/state/DeadServerList.java | 2 +-
.../manager/state/TabletStateChangeIterator.java | 2 +-
.../server/master/balancer/TabletBalancer.java | 4 +-
.../server/replication/ReplicationUtil.java | 2 +-
.../server/security/AuditedSecurityOperation.java | 2 +-
.../server/security/SecurityOperation.java | 2 +-
.../org/apache/accumulo/server/util/FileUtil.java | 20 +-
.../apache/accumulo/server/util/TableInfoUtil.java | 2 +-
.../master/balancer/TableLoadBalancerTest.java | 2 +
.../accumulo/manager/FateServiceHandler.java | 4 +-
.../java/org/apache/accumulo/manager/Manager.java | 59 +-
.../manager/ManagerClientServiceHandler.java | 12 +-
.../accumulo/manager/TabletGroupWatcher.java | 22 +-
.../accumulo/manager/state/SetGoalState.java | 7 +-
.../apache/accumulo/manager/state/TableStats.java | 2 +-
.../tableOps/bulkVer2/PrepBulkImportTest.java | 2 +-
.../java/org/apache/accumulo/monitor/Monitor.java | 4 +-
.../apache/accumulo/monitor/rest/XMLResource.java | 2 +-
.../rest/bulkImports/BulkImportResource.java | 2 +-
.../monitor/rest/manager/ManagerResource.java | 4 +-
.../accumulo/monitor/rest/scans/ScansResource.java | 2 +-
.../monitor/rest/status/StatusResource.java | 2 +-
.../monitor/rest/tables/TablesResource.java | 2 +-
.../rest/tservers/TabletServerResource.java | 2 +-
.../accumulo/monitor/rest/zk/ZKInformation.java | 43 --
.../apache/accumulo/monitor/rest/zk/ZooKeeper.java | 51 --
.../monitor/rest/zk/ZookeeperResource.java | 56 --
.../org/apache/accumulo/monitor/view/WebViews.java | 1 +
.../accumulo/monitor/resources/js/functions.js | 10 -
.../accumulo/monitor/resources/js/overview.js | 31 -
.../apache/accumulo/monitor/templates/modals.ftl | 4 +
.../apache/accumulo/monitor/templates/overview.ftl | 12 +-
.../org/apache/accumulo/monitor/it/WebViewsIT.java | 1 +
.../apache/accumulo/tracer/thrift/Annotation.java | 0
.../apache/accumulo/tracer/thrift/RemoteSpan.java | 0
.../accumulo/tracer/thrift/SpanReceiver.java | 0
.../apache/accumulo/tracer/thrift/TestService.java | 0
.../apache/accumulo/tserver/AssignmentHandler.java | 2 +-
.../org/apache/accumulo/tserver/OnlineTablets.java | 5 +-
.../org/apache/accumulo/tserver/TabletServer.java | 2 +-
.../tserver/TabletServerResourceManager.java | 34 +-
.../accumulo/tserver/UnloadTabletHandler.java | 2 +-
.../tserver/managermessage/ManagerMessage.java | 2 +-
.../tserver/managermessage/SplitReportMessage.java | 4 +-
.../managermessage/TabletStatusMessage.java | 4 +-
.../accumulo/tserver/tablet/CompactableImpl.java | 2 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 12 +-
.../apache/accumulo/tserver/InMemoryMapTest.java | 64 +-
.../java/org/apache/accumulo/shell/ShellUtil.java | 17 +-
.../shell/commands/BulkImportListIterator.java | 2 +-
.../accumulo/shell/commands/ListBulkCommand.java | 4 +-
.../vfs/AccumuloVFSClassLoaderTest.java | 1 -
.../accumulo/test/BulkImportMonitoringIT.java | 2 +-
.../accumulo/test/CountNameNodeOpsBulkIT.java | 2 +-
.../accumulo/test/DetectDeadTabletServersIT.java | 4 +-
.../org/apache/accumulo/test/GetManagerStats.java | 6 +-
.../functional/BalanceAfterCommsFailureIT.java | 4 +-
.../BalanceInPresenceOfOfflineTableIT.java | 4 +-
.../accumulo/test/functional/ManagerApiIT.java | 4 +-
.../test/functional/MetadataMaxFilesIT.java | 4 +-
.../test/functional/SimpleBalancerFairnessIT.java | 4 +-
.../apache/accumulo/test/functional/SummaryIT.java | 12 +-
.../functional/TabletStateChangeIteratorIT.java | 2 +-
.../apache/accumulo/test/manager/MergeStateIT.java | 2 +-
.../test/rpc/thrift/SimpleThriftService.java | 0
187 files changed, 1223 insertions(+), 1413 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 23e52e6,a7554fd..741288d
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -306,15 -298,12 +306,14 @@@ public class TabletMetadata
Objects.requireNonNull(rowIter);
TabletMetadata te = new TabletMetadata();
- ImmutableSortedMap.Builder<Key,Value> kvBuilder = null;
- if (buildKeyValueMap) {
- kvBuilder = ImmutableSortedMap.naturalOrder();
- }
-
- var filesBuilder = ImmutableMap.<StoredTabletFile,DataFileValue>builder();
- var scansBuilder = ImmutableList.<StoredTabletFile>builder();
- var logsBuilder = ImmutableList.<LogEntry>builder();
- var extCompBuilder =
ImmutableMap.<ExternalCompactionId,ExternalCompactionMetadata>builder();
+ final ImmutableSortedMap.Builder<Key,Value> kvBuilder =
+ buildKeyValueMap ? ImmutableSortedMap.naturalOrder() : null;
+
+ final var filesBuilder =
ImmutableMap.<StoredTabletFile,DataFileValue>builder();
+ final var scansBuilder = ImmutableList.<StoredTabletFile>builder();
+ final var logsBuilder = ImmutableList.<LogEntry>builder();
++ final var extCompBuilder =
++
ImmutableMap.<ExternalCompactionId,ExternalCompactionMetadata>builder();
final var loadedFilesBuilder = ImmutableMap.<TabletFile,Long>builder();
ByteSequence row = null;
diff --cc core/src/main/scripts/generate-thrift.sh
index 181e839,1ecfa33..b973481
--- a/core/src/main/scripts/generate-thrift.sh
+++ b/core/src/main/scripts/generate-thrift.sh
@@@ -32,7 -32,7 +32,7 @@@
[[ -z $REQUIRED_THRIFT_VERSION ]] && REQUIRED_THRIFT_VERSION='0.12.0'
[[ -z $INCLUDED_MODULES ]] && INCLUDED_MODULES=(../server/tracer)
[[ -z $BASE_OUTPUT_PACKAGE ]] &&
BASE_OUTPUT_PACKAGE='org.apache.accumulo.core'
- [[ -z $PACKAGES_TO_GENERATE ]] && PACKAGES_TO_GENERATE=(gc manager
tabletserver securityImpl clientImpl dataImpl replication trace compaction)
-[[ -z $PACKAGES_TO_GENERATE ]] && PACKAGES_TO_GENERATE=(gc master manager
tabletserver securityImpl clientImpl dataImpl replication trace)
++[[ -z $PACKAGES_TO_GENERATE ]] && PACKAGES_TO_GENERATE=(gc master manager
tabletserver securityImpl clientImpl dataImpl replication trace compaction)
[[ -z $BUILD_DIR ]] && BUILD_DIR='target'
[[ -z $LANGUAGES_TO_GENERATE ]] && LANGUAGES_TO_GENERATE=(java)
[[ -z $FINAL_DIR ]] && FINAL_DIR='src/main'
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index f74a060,e99c86f..c25dfd8
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -49,13 -43,10 +49,13 @@@ import org.apache.accumulo.core.conf.Pr
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.logging.TabletLogger;
- import org.apache.accumulo.core.master.thrift.TabletLoadState;
+ import org.apache.accumulo.core.manager.thrift.TabletLoadState;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import
org.apache.accumulo.core.spi.compaction.CompactionDispatcher.DispatchParameters;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
