This is an automated email from the ASF dual-hosted git repository. timoninmaxim pushed a commit to branch IGNITE-17177_inc_snapshots in repository https://gitbox.apache.org/repos/asf/ignite.git
commit ebb7ef3f2cc280f2faf66ab4012a69b4b964065f Merge: 66f0e97a5b7 e441d91e793 Author: Maksim Timonin <[email protected]> AuthorDate: Mon Dec 19 17:06:14 2022 +0300 Merge remote-tracking branch 'upstream/master' into IGNITE-17177_inc_snapshots .asf.yaml | 2 +- .github/workflows/commit-check.yml | 2 +- assembly/dependencies-apache-ignite-lgpl.xml | 3 - assembly/dependencies-apache-ignite-slim.xml | 3 - assembly/dependencies-apache-ignite.xml | 3 - assembly/dependencies-visor-console.xml | 132 - assembly/release-apache-ignite-hadoop.xml | 9 - assembly/release-base.xml | 4 - assembly/release-scala.xml | 51 - bin/ignitevisorcmd.bat | 173 -- bin/ignitevisorcmd.sh | 144 - bin/setup-hadoop.bat | 28 - bin/setup-hadoop.sh | 63 - config/visor-cmd/node_startup_by_ssh.sample.ini | 79 - docs/_docs/monitoring-metrics/system-views.adoc | 1 - docs/_docs/snapshots/snapshots.adoc | 21 + docs/_docs/tools/control-script.adoc | 6 +- docs/assets/images/integrations/CDC-design.svg | 2 +- examples/pom.xml | 19 - .../benchmarks/jmh/sql/JmhSqlBenchmark.java | 43 +- .../benchmarks/jmh/tree/IndexFindBenchmark.java | 4 +- .../calcite/CalciteQueryEngineConfiguration.java | 54 + .../query/calcite/CalciteQueryProcessor.java | 28 + .../internal/processors/query/calcite/Query.java | 27 + .../query/calcite/exec/ExchangeServiceImpl.java | 2 + .../query/calcite/exec/ExecutionContext.java | 18 + .../query/calcite/exec/ExecutionService.java | 1 - .../query/calcite/exec/ExecutionServiceImpl.java | 22 + .../query/calcite/exec/IndexFirstLastScan.java | 37 +- .../processors/query/calcite/exec/IndexScan.java | 172 +- .../query/calcite/exec/LogicalRelImplementor.java | 2 +- .../query/calcite/exec/exp/ConverterUtils.java | 13 + .../query/calcite/exec/exp/agg/Accumulators.java | 9 +- .../calcite/exec/exp/agg/AccumulatorsFactory.java | 2 - .../query/calcite/exec/exp/agg/GroupKey.java | 9 +- .../calcite/exec/exp/agg/IterableAccumulator.java} | 10 +- .../query/calcite/exec/rel/AbstractSetOpNode.java | 38 +- .../query/calcite/exec/rel/AggregateNode.java | 80 + .../query/calcite/exec/rel/CollectNode.java | 26 +- .../query/calcite/exec/rel/HashAggregateNode.java | 103 +- .../query/calcite/exec/rel/IndexSpoolNode.java | 6 +- .../query/calcite/exec/rel/IntersectNode.java | 5 + .../query/calcite/exec/rel/MemoryTrackingNode.java | 64 + .../query/calcite/exec/rel/MinusNode.java | 5 + .../query/calcite/exec/rel/NestedLoopJoinNode.java | 5 +- .../query/calcite/exec/rel/SortAggregateNode.java | 43 +- .../query/calcite/exec/rel/SortNode.java | 20 +- .../query/calcite/exec/rel/TableSpoolNode.java | 4 +- .../exec/tracker/ExecutionNodeMemoryTracker.java | 91 + .../calcite/exec/tracker/GlobalMemoryTracker.java | 63 + .../query/calcite/exec/tracker/MemoryTracker.java} | 19 +- .../calcite/exec/tracker/NoOpMemoryTracker.java} | 38 +- .../calcite/exec/tracker/NoOpRowTracker.java} | 31 +- .../calcite/exec/tracker/ObjectSizeCalculator.java | 332 +++ .../calcite/exec/tracker/QueryMemoryTracker.java | 95 + .../query/calcite/exec/tracker/RowTracker.java} | 24 +- .../query/calcite/externalize/RelJsonReader.java | 3 + .../query/calcite/rel/AbstractIndexScan.java | 27 +- .../query/calcite/rel/IgniteCollect.java | 21 + .../rel/IgniteCorrelatedNestedLoopJoin.java | 29 + .../processors/query/calcite/rel/IgniteFilter.java | 83 +- .../query/calcite/rel/IgniteIndexCount.java | 28 +- .../processors/query/calcite/rel/IgniteLimit.java | 15 +- .../rel/agg/IgniteColocatedAggregateBase.java | 12 +- .../calcite/rel/set/IgniteColocatedSetOp.java | 6 +- .../query/calcite/rule/IndexCountRule.java | 54 +- .../query/calcite/schema/CacheIndexImpl.java | 76 +- .../query/calcite/schema/IgniteIndex.java | 8 +- .../query/calcite/schema/SystemViewIndexImpl.java | 9 +- .../query/calcite/trait/DistributionFunction.java | 66 + .../query/calcite/trait/DistributionTrait.java | 5 + .../query/calcite/trait/IgniteDistributions.java | 15 + .../processors/query/calcite/trait/TraitUtils.java | 4 + .../query/calcite/type/IgniteTypeFactory.java | 18 +- .../processors/query/calcite/util/TypeUtils.java | 50 +- .../processors/query/calcite/QueryChecker.java | 13 + .../calcite/exec/LogicalRelImplementorTest.java | 6 +- .../query/calcite/exec/RuntimeSortedIndexTest.java | 2 + .../calcite/exec/rel/AbstractExecutionTest.java | 2 + .../calcite/exec/tracker/MemoryTrackerTest.java | 256 ++ .../integration/AbstractBasicIntegrationTest.java | 25 +- .../integration/AggregatesIntegrationTest.java | 36 +- .../integration/CorrelatesIntegrationTest.java | 21 + .../query/calcite/integration/FunctionsTest.java | 28 +- .../integration/IndexScanlIntegrationTest.java | 152 +- .../LimitOffsetIntegrationTest.java} | 169 +- .../integration/LocalDateTimeSupportTest.java | 290 +++ .../integration/MemoryQuotasIntegrationTest.java | 405 +++ .../query/calcite/logical/SqlScriptRunner.java | 9 + .../query/calcite/planner/AbstractPlannerTest.java | 3 +- .../planner/CorrelatedSubqueryPlannerTest.java | 87 + .../calcite/planner/HashAggregatePlannerTest.java | 7 +- .../planner/InlineIndexScanPlannerTest.java | 76 + .../calcite/planner/LimitOffsetPlannerTest.java | 6 + .../calcite/planner/MergeJoinPlannerTest.java | 3 +- .../query/calcite/planner/PlannerTest.java | 298 +-- .../query/calcite/planner/TestTable.java | 44 +- .../ignite/testsuites/IgniteCalciteTestSuite.java | 2 + .../ignite/testsuites/IntegrationTestSuite.java | 10 +- .../apache/ignite/testsuites/PlannerTestSuite.java | 2 + .../src/test/sql/cast/test_boolean_cast.test | 21 + .../test/sql/cast/test_boolean_cast.test_ignore | 160 -- .../src/test/sql/types/collections/array.test | 7 +- .../src/test/sql/types/collections/array_agg.test | 4 +- .../src/test/sql/types/collections/map.test | 3 + .../ClientTcpUnreachableMultiNodeSelfTest.java | 4 - .../internal/jdbc2/JdbcConnectionSelfTest.java | 32 - .../rest/JettyRestProcessorAbstractSelfTest.java | 603 +---- .../ignite/jdbc/thin/JdbcThinMetadataSelfTest.java | 1 - .../DiskPageCompressionIntegrationTest.java | 8 +- .../ignite/internal/commandline/TaskExecutor.java | 2 +- .../commandline/cache/CheckIndexInlineSizes.java | 2 +- .../internal/commandline/cache/IdleVerify.java | 25 +- .../consistency/ConsistencyCommand.java | 2 +- .../commandline/snapshot/SnapshotCheckCommand.java | 4 +- .../commandline/systemview/SystemViewCommand.java | 116 +- .../systemview/SystemViewCommandArg.java | 15 +- .../util/GridCommandHandlerAbstractTest.java | 5 - .../util/GridCommandHandlerClusterByClassTest.java | 33 + .../GridCommandHandlerConsistencyCountersTest.java | 61 +- .../apache/ignite/util/GridCommandHandlerTest.java | 103 +- .../apache/ignite/util/SystemViewCommandTest.java | 142 +- .../java/org/apache/ignite/DataStorageMetrics.java | 266 -- .../apache/ignite/DataStorageMetricsAdapter.java | 114 - .../src/main/java/org/apache/ignite/Ignite.java | 13 - .../main/java/org/apache/ignite/IgniteCache.java | 9 +- .../main/java/org/apache/ignite/IgniteCluster.java | 2 +- .../org/apache/ignite/IgniteSystemProperties.java | 26 +- .../src/main/java/org/apache/ignite/Ignition.java | 28 - .../java/org/apache/ignite/PersistenceMetrics.java | 124 - .../java/org/apache/ignite/cache/QueryEntity.java | 57 +- .../java/org/apache/ignite/cdc/CdcConsumer.java | 10 + .../org/apache/ignite/cluster/ClusterGroup.java | 16 - .../org/apache/ignite/cluster/ClusterNode.java | 19 - .../configuration/DataStorageConfiguration.java | 13 +- .../ignite/configuration/IgniteConfiguration.java | 49 +- .../ClusterLocalNodeMetricsMXBeanImpl.java | 2 +- .../org/apache/ignite/internal/GridDiagnostic.java | 6 +- .../apache/ignite/internal/GridKernalContext.java | 7 - .../ignite/internal/GridKernalContextImpl.java | 14 +- .../org/apache/ignite/internal/IgniteKernal.java | 102 +- .../ignite/internal/IgniteNodeAttributes.java | 3 - .../org/apache/ignite/internal/IgnitionEx.java | 39 - .../org/apache/ignite/internal/SkipDaemon.java | 32 - .../internal/StripedExecutorMXBeanAdapter.java | 96 - .../ignite/internal/ThreadPoolMXBeanAdapter.java | 132 - .../query/index/IndexQueryCriteriaClosure.java | 3 +- .../cache/query/index/IndexSingleRangeQuery.java | 14 +- ...exSearchRowImpl.java => IndexPlainRowImpl.java} | 18 +- .../cache/query/index/sorted/IndexRow.java | 6 +- .../cache/query/index/sorted/IndexRowImpl.java | 13 +- .../index/sorted/client/ClientIndexFactory.java | 2 +- .../index/sorted/client/ClientInlineIndex.java | 18 +- .../index/sorted/inline/IndexQueryContext.java | 21 + .../query/index/sorted/inline/InlineIndex.java | 6 + .../query/index/sorted/inline/InlineIndexImpl.java | 12 +- .../index/sorted/inline/InlineIndexKeyType.java | 24 +- .../query/index/sorted/inline/InlineIndexTree.java | 4 +- .../index/sorted/inline/InlineRecommender.java | 2 +- .../inline/types/BytesInlineIndexKeyType.java | 7 +- .../inline/types/NullableInlineIndexKeyType.java | 40 +- .../types/ObjectByteArrayInlineIndexKeyType.java | 4 +- .../inline/types/StringInlineIndexKeyType.java | 10 +- .../types/StringNoCompareInlineIndexKeyType.java | 7 +- .../org/apache/ignite/internal/cdc/CdcMain.java | 7 + .../ignite/internal/cdc/WalRecordsConsumer.java | 10 + .../ignite/internal/client/GridClientCompute.java | 11 +- .../ignite/internal/client/GridClientNode.java | 8 - .../client/impl/GridClientComputeImpl.java | 12 +- .../internal/client/impl/GridClientNodeImpl.java | 6 - .../client/thin/ClientClusterNodeImpl.java | 9 - .../internal/cluster/ClusterGroupAdapter.java | 38 +- .../internal/cluster/DetachedClusterNode.java | 5 - .../internal/cluster/IgniteClusterAsyncImpl.java | 5 - .../ignite/internal/cluster/IgniteClusterImpl.java | 4 +- .../ignite/internal/cluster/IgniteKillTask.java | 15 +- .../ignite/internal/jdbc2/JdbcConnection.java | 4 - .../internal/managers/GridManagerAdapter.java | 13 - .../internal/managers/IgniteMBeansManager.java | 2 - .../managers/checkpoint/GridCheckpointManager.java | 5 - .../managers/collision/GridCollisionManager.java | 5 - .../internal/managers/discovery/DiscoCache.java | 12 - .../managers/discovery/GridDiscoveryManager.java | 167 +- .../managers/encryption/GridEncryptionManager.java | 6 +- .../managers/encryption/GroupKeyChangeProcess.java | 2 +- .../eventstorage/GridEventStorageManager.java | 9 +- .../managers/indexing/GridIndexingManager.java | 8 - .../systemview/walker/ClusterNodeViewWalker.java | 20 +- .../internal/plugin/IgniteLogInfoProviderImpl.java | 18 - .../IgniteAuthenticationProcessor.java | 4 +- .../processors/cache/CacheGroupMetricsImpl.java | 8 + .../cache/CacheGroupMetricsMXBeanImpl.java | 183 -- .../processors/cache/CacheOperationContext.java | 11 +- .../processors/cache/ClusterCachesInfo.java | 34 +- .../internal/processors/cache/ExchangeContext.java | 22 + .../processors/cache/GridCacheAdapter.java | 12 +- .../processors/cache/GridCacheAffinityManager.java | 35 + .../cache/GridCacheDeploymentManager.java | 13 +- .../processors/cache/GridCacheGateway.java | 5 +- .../cache/GridCachePartitionExchangeManager.java | 2 +- .../processors/cache/GridCacheProcessor.java | 29 +- .../processors/cache/GridCacheProxyImpl.java | 10 +- .../processors/cache/GridCacheTtlManager.java | 2 +- .../internal/processors/cache/GridCacheUtils.java | 2 +- .../processors/cache/GridLocalConfigManager.java | 5 - .../cache/IgniteCacheOffheapManager.java | 5 + .../cache/IgniteCacheOffheapManagerImpl.java | 5 + .../processors/cache/PartitionUpdateCounter.java | 7 + .../cache/PartitionUpdateCounterDebugWrapper.java | 5 + .../cache/PartitionUpdateCounterErrorWrapper.java | 5 + .../cache/PartitionUpdateCounterTrackingImpl.java | 10 +- .../cache/PartitionUpdateCounterVolatileImpl.java | 5 + .../cache/ValidationOnNodeJoinUtils.java | 27 +- .../internal/processors/cache/WalStateManager.java | 2 +- .../datastructures/CacheDataStructuresManager.java | 3 +- .../cache/distributed/dht/GridDhtCacheAdapter.java | 30 +- .../dht/GridDhtTransactionalCacheAdapter.java | 2 +- .../distributed/dht/atomic/GridDhtAtomicCache.java | 2 +- .../dht/preloader/GridDhtPartitionDemander.java | 6 +- .../dht/preloader/GridDhtPartitionSupplier.java | 4 +- .../preloader/GridDhtPartitionsExchangeFuture.java | 12 +- .../dht/preloader/InitNewCoordinatorFuture.java | 2 +- .../dht/preloader/latch/ExchangeLatchManager.java | 4 +- .../dht/topology/GridClientPartitionTopology.java | 4 +- .../dht/topology/GridDhtLocalPartition.java | 8 + .../dht/topology/GridDhtPartitionTopologyImpl.java | 6 +- .../dht/topology/PartitionsEvictManager.java | 2 +- .../cache/persistence/DataStorageMetricsImpl.java | 396 ++- .../persistence/DataStorageMetricsSnapshot.java | 325 --- .../GridCacheDatabaseSharedManager.java | 20 - .../cache/persistence/GridCacheOffheapManager.java | 17 +- .../IgniteCacheDatabaseSharedManager.java | 8 - .../checkpoint/CheckpointPagesWriter.java | 1 - .../persistence/checkpoint/CheckpointWorkflow.java | 2 +- .../persistence/file/FilePageStoreManager.java | 46 +- .../persistence/filename/PdsFolderResolver.java | 2 +- .../snapshot/DataStreamerUpdatesHandler.java | 8 +- .../snapshot/IgniteSnapshotManager.java | 408 ++- .../persistence/snapshot/SnapshotFutureTask.java | 118 +- .../persistence/snapshot/SnapshotHandler.java | 4 +- .../snapshot/SnapshotHandlerContext.java | 4 +- .../persistence/snapshot/SnapshotMetadata.java | 23 +- .../snapshot/SnapshotOperationRequest.java | 21 +- .../SnapshotPartitionsQuickVerifyHandler.java | 4 +- .../SnapshotPartitionsVerifyTaskResult.java | 31 + .../snapshot/SnapshotRestoreProcess.java | 8 +- ...xception.java => SnapshotWarningException.java} | 6 +- .../cache/persistence/tree/BPlusTree.java | 63 +- .../persistence/wal/FileWriteAheadLogManager.java | 33 +- .../wal/aware/SegmentArchiveSizeStorage.java | 20 +- .../cache/persistence/wal/aware/SegmentAware.java | 1 + .../wal/aware/SegmentCurrentStateStorage.java | 10 +- .../wal/reader/StandaloneGridKernalContext.java | 5 - .../cache/transactions/IgniteTxHandler.java | 35 +- .../processors/cluster/ClusterProcessor.java | 3 - .../cluster/GridClusterStateProcessor.java | 43 +- .../autoadjust/BaselineTopologyUpdater.java | 2 +- .../continuous/GridContinuousProcessor.java | 17 +- .../datastreamer/DataStreamProcessor.java | 6 - .../datastructures/DataStructuresProcessor.java | 3 - .../datastructures/GridCacheQueueAdapter.java | 4 +- .../internal/processors/job/GridJobProcessor.java | 2 - .../persistence/DistributedMetaStorageImpl.java | 2 +- .../processors/odbc/ClientListenerProcessor.java | 8 - .../processors/platform/PlatformContextImpl.java | 1 - .../ClientClusterGroupGetNodesDetailsResponse.java | 2 +- .../platform/cluster/PlatformClusterGroup.java | 79 - .../platform/utils/PlatformConfigurationUtils.java | 4 - .../internal/processors/pool/PoolProcessor.java | 72 - .../processors/query/GridQueryIndexing.java | 13 - .../processors/query/GridQueryProcessor.java | 3 +- .../internal/processors/query/QueryField.java | 24 + .../internal/processors/query/QuerySchema.java | 4 +- .../processors/query/QueryTypeDescriptorImpl.java | 11 +- .../internal/processors/query/QueryUtils.java | 53 +- .../query/stat/IgniteStatisticsManagerImpl.java | 2 +- .../internal/processors/rest/GridRestCommand.java | 3 - .../processors/rest/GridRestProcessor.java | 8 +- .../memory/MemoryMetricsCommandHandler.java | 12 +- .../handlers/top/GridTopologyCommandHandler.java | 3 +- .../processors/service/IgniteServiceProcessor.java | 2 - .../processors/task/GridTaskProcessor.java | 5 +- .../apache/ignite/internal/util/GridUnsafe.java | 10 + .../apache/ignite/internal/util/IgniteUtils.java | 7 +- .../util/tostring/GridToStringBuilder.java | 16 +- .../apache/ignite/internal/visor/VisorEither.java | 103 - .../internal/visor/binary/VisorBinaryMetadata.java | 138 - .../binary/VisorBinaryMetadataCollectorTask.java | 74 - .../VisorBinaryMetadataCollectorTaskArg.java | 71 - .../VisorBinaryMetadataCollectorTaskResult.java | 87 - .../visor/binary/VisorBinaryMetadataField.java | 101 - .../ignite/internal/visor/cache/VisorCache.java | 341 --- .../visor/cache/VisorCacheAffinityNodeTask.java | 71 - .../visor/cache/VisorCacheAffinityNodeTaskArg.java | 86 - .../visor/cache/VisorCacheAggregatedMetrics.java | 628 ----- .../internal/visor/cache/VisorCacheClearTask.java | 146 -- .../visor/cache/VisorCacheClearTaskArg.java | 72 - .../visor/cache/VisorCacheClearTaskResult.java | 85 - .../internal/visor/cache/VisorCacheLoadTask.java | 102 - .../visor/cache/VisorCacheLoadTaskArg.java | 101 - .../visor/cache/VisorCacheLostPartitionsTask.java | 88 - .../cache/VisorCacheLostPartitionsTaskArg.java | 81 - .../cache/VisorCacheLostPartitionsTaskResult.java | 74 - .../visor/cache/VisorCacheMetadataTask.java | 87 - .../visor/cache/VisorCacheMetadataTaskArg.java | 72 - .../internal/visor/cache/VisorCacheMetrics.java | 812 ------ .../cache/VisorCacheMetricsCollectorTask.java | 136 - .../cache/VisorCacheMetricsCollectorTaskArg.java | 87 - .../internal/visor/cache/VisorCacheModifyTask.java | 115 - .../visor/cache/VisorCacheModifyTaskArg.java | 120 - .../visor/cache/VisorCacheModifyTaskResult.java | 101 - .../visor/cache/VisorCacheNamesCollectorTask.java | 90 - .../cache/VisorCacheNamesCollectorTaskResult.java | 88 - .../internal/visor/cache/VisorCacheNodesTask.java | 76 - .../visor/cache/VisorCacheNodesTaskArg.java | 72 - .../internal/visor/cache/VisorCachePartitions.java | 100 - .../visor/cache/VisorCachePartitionsTask.java | 147 -- .../visor/cache/VisorCachePartitionsTaskArg.java | 72 - .../visor/cache/VisorCacheRebalanceTask.java | 86 - .../visor/cache/VisorCacheRebalanceTaskArg.java | 73 - .../cache/VisorCacheResetLostPartitionsTask.java | 67 - .../VisorCacheResetLostPartitionsTaskArg.java | 81 - .../visor/cache/VisorCacheResetMetricsTask.java | 69 - .../visor/cache/VisorCacheResetMetricsTaskArg.java | 72 - .../visor/cache/VisorCacheSqlIndexMetadata.java | 115 - .../visor/cache/VisorCacheSqlMetadata.java | 162 -- .../internal/visor/cache/VisorCacheStartTask.java | 112 - .../visor/cache/VisorCacheStartTaskArg.java | 100 - .../cache/VisorCacheToggleStatisticsTask.java | 72 - .../cache/VisorCacheToggleStatisticsTaskArg.java | 87 - .../internal/visor/cache/VisorMemoryMetrics.java | 352 --- .../internal/visor/cache/VisorModifyCacheMode.java | 47 - .../internal/visor/cache/VisorPartitionMap.java | 110 - .../compute/VisorComputeMonitoringHolder.java | 121 - .../compute/VisorComputeResetMetricsTask.java | 67 - .../compute/VisorComputeToggleMonitoringTask.java | 112 - .../VisorComputeToggleMonitoringTaskArg.java | 86 - .../internal/visor/compute/VisorGatewayTask.java | 475 ---- .../internal/visor/debug/VisorThreadDumpTask.java | 77 - .../visor/debug/VisorThreadDumpTaskResult.java | 88 - .../internal/visor/debug/VisorThreadInfo.java | 348 --- .../internal/visor/debug/VisorThreadLockInfo.java | 93 - .../visor/debug/VisorThreadMonitorInfo.java | 100 - .../visor/event/VisorGridDeploymentEvent.java | 106 - .../visor/event/VisorGridDiscoveryEvent.java | 152 -- .../internal/visor/event/VisorGridEvent.java | 162 -- .../internal/visor/event/VisorGridEventsLost.java | 54 - .../internal/visor/event/VisorGridJobEvent.java | 151 -- .../internal/visor/event/VisorGridTaskEvent.java | 151 -- .../ignite/internal/visor/file/VisorFileBlock.java | 144 - .../internal/visor/file/VisorFileBlockTask.java | 89 - .../internal/visor/file/VisorFileBlockTaskArg.java | 114 - .../visor/file/VisorFileBlockTaskResult.java | 87 - .../visor/file/VisorLatestTextFilesTask.java | 97 - .../visor/file/VisorLatestTextFilesTaskArg.java | 86 - .../ignite/internal/visor/igfs/VisorIgfs.java | 117 - .../internal/visor/igfs/VisorIgfsEndpoint.java | 134 - .../internal/visor/igfs/VisorIgfsFormatTask.java | 67 - .../visor/igfs/VisorIgfsFormatTaskArg.java | 73 - .../internal/visor/igfs/VisorIgfsMetrics.java | 270 -- .../ignite/internal/visor/igfs/VisorIgfsMode.java | 72 - .../internal/visor/igfs/VisorIgfsProfiler.java | 88 - .../visor/igfs/VisorIgfsProfilerClearTask.java | 70 - .../visor/igfs/VisorIgfsProfilerClearTaskArg.java | 73 - .../igfs/VisorIgfsProfilerClearTaskResult.java | 86 - .../visor/igfs/VisorIgfsProfilerEntry.java | 284 -- .../internal/visor/igfs/VisorIgfsProfilerTask.java | 71 - .../visor/igfs/VisorIgfsProfilerTaskArg.java | 73 - .../igfs/VisorIgfsProfilerUniformityCounters.java | 233 -- .../visor/igfs/VisorIgfsResetMetricsTask.java | 67 - .../visor/igfs/VisorIgfsResetMetricsTaskArg.java | 74 - .../visor/igfs/VisorIgfsSamplingStateTask.java | 67 - .../visor/igfs/VisorIgfsSamplingStateTaskArg.java | 87 - .../ignite/internal/visor/log/VisorLogFile.java | 112 - .../internal/visor/log/VisorLogSearchResult.java | 192 -- .../internal/visor/log/VisorLogSearchTask.java | 239 -- .../internal/visor/log/VisorLogSearchTaskArg.java | 114 - .../visor/log/VisorLogSearchTaskResult.java | 92 - .../ignite/internal/visor/misc/VisorAckTask.java | 75 - .../internal/visor/misc/VisorAckTaskArg.java | 72 - .../visor/misc/VisorChangeGridActiveStateTask.java | 67 - .../misc/VisorChangeGridActiveStateTaskArg.java | 71 - .../visor/misc/VisorLatestVersionTask.java | 63 - .../ignite/internal/visor/misc/VisorNopTask.java | 98 - .../visor/misc/VisorResolveHostNameTask.java | 107 - .../visor/node/VisorAffinityTopologyVersion.java | 87 - .../visor/node/VisorAtomicConfiguration.java | 101 - .../visor/node/VisorBasicConfiguration.java | 551 ---- .../visor/node/VisorBinaryConfiguration.java | 131 - .../visor/node/VisorBinaryTypeConfiguration.java | 150 -- .../visor/node/VisorCacheKeyConfiguration.java | 108 - .../VisorCacheRebalanceCollectorJobResult.java | 91 - .../node/VisorCacheRebalanceCollectorTask.java | 199 -- .../node/VisorCacheRebalanceCollectorTaskArg.java | 54 - .../VisorCacheRebalanceCollectorTaskResult.java | 92 - .../node/VisorClientConnectorConfiguration.java | 281 -- .../visor/node/VisorDataRegionConfiguration.java | 238 -- .../visor/node/VisorDataStorageConfiguration.java | 491 ---- .../visor/node/VisorExecutorConfiguration.java | 108 - .../node/VisorExecutorServiceConfiguration.java | 269 -- .../visor/node/VisorGridConfiguration.java | 470 ---- .../visor/node/VisorHadoopConfiguration.java | 127 - .../visor/node/VisorIgfsConfiguration.java | 320 --- .../visor/node/VisorLifecycleConfiguration.java | 78 - .../visor/node/VisorMemoryConfiguration.java | 167 -- .../visor/node/VisorMemoryPolicyConfiguration.java | 159 -- .../visor/node/VisorMetricsConfiguration.java | 100 - .../visor/node/VisorMvccConfiguration.java | 94 - .../visor/node/VisorNodeBaselineStatus.java | 47 - .../node/VisorNodeConfigurationCollectorJob.java | 47 - .../visor/node/VisorNodeDataCollectorJob.java | 308 --- .../node/VisorNodeDataCollectorJobResult.java | 375 --- .../visor/node/VisorNodeDataCollectorTask.java | 136 - .../visor/node/VisorNodeDataCollectorTaskArg.java | 238 -- .../node/VisorNodeDataCollectorTaskResult.java | 374 --- .../visor/node/VisorNodeEventsCollectorTask.java | 220 -- .../node/VisorNodeEventsCollectorTaskArg.java | 163 -- .../internal/visor/node/VisorNodeGcTask.java | 102 - .../internal/visor/node/VisorNodeGcTaskResult.java | 85 - .../internal/visor/node/VisorNodePingTask.java | 81 - .../internal/visor/node/VisorNodePingTaskArg.java | 73 - .../visor/node/VisorNodePingTaskResult.java | 99 - .../internal/visor/node/VisorNodeRestartTask.java | 80 - .../internal/visor/node/VisorNodeStopTask.java | 80 - .../visor/node/VisorNodeSuppressedErrors.java | 89 - .../visor/node/VisorNodeSuppressedErrorsTask.java | 110 - .../node/VisorNodeSuppressedErrorsTaskArg.java | 74 - .../visor/node/VisorPeerToPeerConfiguration.java | 104 - .../visor/node/VisorPersistenceMetrics.java | 463 ---- .../node/VisorPersistentStoreConfiguration.java | 306 --- .../visor/node/VisorRestConfiguration.java | 366 --- .../visor/node/VisorSegmentationConfiguration.java | 144 - .../visor/node/VisorServiceConfiguration.java | 175 -- .../internal/visor/node/VisorSpiDescription.java | 89 - .../visor/node/VisorSpisConfiguration.java | 256 -- .../visor/node/VisorSqlConnectorConfiguration.java | 170 -- .../internal/visor/node/VisorSuppressedError.java | 147 -- .../visor/node/VisorTransactionConfiguration.java | 159 -- .../internal/visor/query/VisorQueryCancelTask.java | 74 - .../visor/query/VisorQueryCancelTaskArg.java | 71 - .../visor/query/VisorQueryCleanupTask.java | 134 - .../visor/query/VisorQueryCleanupTaskArg.java | 75 - .../visor/query/VisorQueryDetailMetrics.java | 205 -- .../VisorQueryDetailMetricsCollectorTask.java | 145 -- .../VisorQueryDetailMetricsCollectorTaskArg.java | 71 - .../visor/query/VisorQueryFetchFirstPageTask.java | 107 - .../internal/visor/query/VisorQueryField.java | 132 - .../internal/visor/query/VisorQueryHolder.java | 171 -- .../internal/visor/query/VisorQueryMetrics.java | 125 - .../visor/query/VisorQueryNextPageTask.java | 91 - .../visor/query/VisorQueryNextPageTaskArg.java | 86 - .../internal/visor/query/VisorQueryPingTask.java | 88 - .../visor/query/VisorQueryPingTaskResult.java | 50 - .../query/VisorQueryResetDetailMetricsTask.java | 73 - .../visor/query/VisorQueryResetMetricsTask.java | 75 - .../visor/query/VisorQueryResetMetricsTaskArg.java | 72 - .../internal/visor/query/VisorQueryResult.java | 152 -- .../visor/query/VisorQueryScanRegexFilter.java | 60 - .../internal/visor/query/VisorQueryTask.java | 91 - .../internal/visor/query/VisorQueryTaskArg.java | 249 -- .../internal/visor/query/VisorQueryUtils.java | 519 ---- .../query/VisorRunningQueriesCollectorTask.java | 101 - .../query/VisorRunningQueriesCollectorTaskArg.java | 71 - .../internal/visor/query/VisorRunningQuery.java | 175 -- .../internal/visor/query/VisorScanQueryTask.java | 83 - .../visor/query/VisorScanQueryTaskArg.java | 157 -- .../visor/service/VisorServiceDescriptor.java | 166 -- .../internal/visor/service/VisorServiceTask.java | 77 - .../visor/snapshot/VisorSnapshotCheckTask.java | 12 +- .../visor/systemview/VisorSystemViewTask.java | 33 +- .../systemview/VisorSystemViewTaskResult.java | 29 +- .../internal/visor/util/VisorEventMapper.java | 147 -- .../ignite/internal/visor/util/VisorMimeTypes.java | 1019 -------- .../ignite/internal/visor/util/VisorTaskUtils.java | 920 ------- .../visor/verify/VisorIdleAnalyzeTask.java | 124 - .../visor/verify/VisorIdleAnalyzeTaskArg.java | 88 - .../visor/verify/VisorIdleAnalyzeTaskResult.java | 76 - .../ignite/mxbean/CacheGroupMetricsMXBean.java | 212 -- .../ignite/mxbean/DataStorageMetricsMXBean.java | 206 -- .../ignite/mxbean/PersistenceMetricsMXBean.java | 116 - .../ignite/mxbean/StripedExecutorMXBean.java | 100 - .../org/apache/ignite/mxbean/ThreadPoolMXBean.java | 167 -- .../plugin/segmentation/SegmentationPolicy.java | 5 +- .../org/apache/ignite/spi/IgniteSpiAdapter.java | 8 +- .../org/apache/ignite/spi/IgniteSpiContext.java | 12 - .../tcp/internal/GridNioServerWrapper.java | 2 +- .../spi/discovery/isolated/IsolatedNode.java | 5 - .../ignite/spi/discovery/tcp/ClientImpl.java | 14 +- .../ignite/spi/discovery/tcp/ServerImpl.java | 9 +- .../ignite/spi/discovery/tcp/TcpDiscoverySpi.java | 6 - .../discovery/tcp/internal/TcpDiscoveryNode.java | 21 - .../spi/systemview/view/ClusterNodeView.java | 15 +- .../main/resources/META-INF/classnames.properties | 900 +++---- .../ignite/cdc/CdcNonDefaultWorkDirTest.java | 109 + .../java/org/apache/ignite/cdc/WalForCdcTest.java | 57 +- .../cluster/GridClusterStateChangeSelfTest.java | 3 +- .../ignite/internal/ClusterGroupSelfTest.java | 28 - ...rocessorCheckGlobalStateComputeRequestTest.java | 69 - .../internal/GridDiscoveryEventSelfTest.java | 53 - .../ignite/internal/GridDiscoverySelfTest.java | 5 - .../org/apache/ignite/internal/GridMBeansTest.java | 27 +- .../IgniteSlowClientDetectionSelfTest.java | 4 + ...inaryConfigurationCustomSerializerSelfTest.java | 10 +- .../internal/client/thin/ClusterGroupTest.java | 1 - .../internal/metric/IoStatisticsSelfTest.java | 47 +- .../ignite/internal/metric/SystemViewSelfTest.java | 1 - .../processors/cache/CacheGroupMetricsTest.java | 90 +- .../cache/CacheGroupsMetricsRebalanceTest.java | 17 - .../cache/CacheMetricsForClusterGroupSelfTest.java | 15 - .../processors/cache/CacheMetricsManageTest.java | 2 +- .../cache/GridCacheDaemonNodeAbstractSelfTest.java | 185 -- ...ridProjectionForCachesOnDaemonNodeSelfTest.java | 134 - .../cache/IgniteDaemonNodeMarshallerCacheTest.java | 189 -- .../cache/IgniteDynamicCacheStartSelfTest.java | 39 - .../GridCacheDaemonNodePartitionedSelfTest.java | 31 - .../HistoricalRebalanceCheckpointTest.java | 453 ++++ .../dht/topology/EvictPartitionInLogTest.java | 34 +- .../GridCacheDaemonNodeReplicatedSelfTest.java | 31 - .../IgniteDataStorageMetricsSelfTest.java | 30 +- .../db/file/IgnitePdsThreadInterruptionTest.java | 213 +- .../db/wal/IgniteWalRebalanceLoggingTest.java | 8 +- .../db/wal/WalDeletionArchiveAbstractTest.java | 106 + .../snapshot/AbstractSnapshotSelfTest.java | 4 +- .../snapshot/EncryptedSnapshotTest.java | 37 +- .../snapshot/IgniteClusterSnapshotCheckTest.java | 18 +- .../snapshot/IgniteClusterSnapshotDeltaTest.java | 220 ++ .../IgniteClusterSnapshotRestoreSelfTest.java | 3 +- .../snapshot/IgniteClusterSnapshotSelfTest.java | 25 + .../IgniteClusterSnapshotStreamerTest.java | 262 +- .../snapshot/IgniteSnapshotConsistencyTest.java | 310 +++ .../IgniteSnapshotRestoreFromRemoteTest.java | 4 +- .../standbycluster/IgniteStandByClusterTest.java | 72 - .../wal/WalEnableDisableWithNodeShutdownTest.java | 2 +- .../wal/WalEnableDisableWithRestartsTest.java | 2 +- .../persistence/wal/aware/SegmentAwareTest.java | 25 + .../transactions/AtomicOperationsInTxTest.java | 149 +- .../database/IgniteDbPutGetAbstractTest.java | 6 +- .../processors/query/DummyQueryIndexing.java | 5 - .../processors/rest/RestProtocolStartTest.java | 2 + .../daemon/DaemonNodeBasicSecurityTest.java | 55 - .../AccessToClassesInsideInternalPackageTest.java | 2 +- .../util/tostring/GridToStringBuilderSelfTest.java | 16 +- .../ignite/p2p/GridP2PClassLoadingSelfTest.java | 5 - .../ignite/platform/PlatformProcessUtils.java | 9 +- .../apache/ignite/platform/PlatformTestUtils.java} | 16 +- .../IgniteDiscoveryCacheReuseSelfTest.java | 2 +- .../IgniteClientReconnectMassiveShutdownTest.java | 4 + .../tcp/TcpClientDiscoverySpiSelfTest.java | 4 + .../tcp/TcpDiscoverySegmentationPolicyTest.java | 73 +- .../spi/discovery/tcp/TcpDiscoverySelfTest.java | 3 +- .../GridInternalTasksLoadBalancingSelfTest.java | 23 - .../ignite/testframework/GridSpiTestContext.java | 12 - .../apache/ignite/testframework/GridTestNode.java | 5 - .../testframework/junits/GridAbstractTest.java | 2 - .../ignite/testframework/junits/IgniteMock.java | 13 - .../junits/common/GridCommonAbstractTest.java | 3 - .../junits/multijvm/IgniteClusterProcessProxy.java | 5 - .../junits/multijvm/IgniteProcessProxy.java | 13 - .../ignite/testsuites/IgniteBasicTestSuite.java | 6 - .../ignite/testsuites/IgniteCacheTestSuite3.java | 6 - .../ignite/testsuites/IgnitePdsTestSuite2.java | 5 + .../ignite/testsuites/IgniteSnapshotTestSuite.java | 4 + .../ignite/testsuites/SecurityTestSuite.java | 2 - ...ridCommandHandlerClusterByClassTest_help.output | 8 +- ...andHandlerClusterByClassWithSSLTest_help.output | 8 +- .../internal/processors/query/h2/H2Utils.java | 40 - .../processors/query/h2/IgniteH2Indexing.java | 17 - .../processors/query/h2/database/H2TreeIndex.java | 4 +- .../cache/CacheGroupMetricsWithIndexTest.java | 10 +- .../IgniteClusterSnapshotCheckWithIndexesTest.java | 9 +- ...ntConnectorConfigurationValidationSelfTest.java | 23 - .../processors/query/QueryEntityAliasesTest.java | 227 ++ .../processors/query/SqlSystemViewsSelfTest.java | 36 +- .../h2/H2ColumnTypeConversionCheckSelfTest.java | 57 - .../IgniteBinaryCacheQueryTestSuite3.java | 2 - .../testsuites/IgniteCacheQuerySelfTestSuite6.java | 4 +- .../cpp/core-test/src/cluster_group_test.cpp | 13 - .../cpp/core-test/src/cluster_node_test.cpp | 8 +- .../platforms/cpp/core-test/src/cluster_test.cpp | 13 - .../platforms/cpp/core-test/src/ignition_test.cpp | 12 +- .../core/include/ignite/cluster/cluster_group.h | 7 - .../cpp/core/include/ignite/cluster/cluster_node.h | 7 - .../ignite/impl/cluster/cluster_group_impl.h | 7 - .../ignite/impl/cluster/cluster_node_impl.h | 10 - .../cpp/core/src/cluster/cluster_group.cpp | 5 - .../cpp/core/src/cluster/cluster_node.cpp | 5 - modules/platforms/cpp/core/src/ignition.cpp | 19 + .../core/src/impl/cluster/cluster_group_impl.cpp | 13 - .../core/src/impl/cluster/cluster_node_impl.cpp | 8 +- .../ApiParity/DataStorageMetricsParityTest.cs | 62 - .../Cache/DataStorageMetricsTest.cs | 106 - .../Cache/PersistenceTest.cs | 12 - .../Cache/PersistentStoreTestObsolete.cs | 5 - .../Cache/Query/Linq/CacheLinqTest.Base.cs | 4 +- .../Cache/Query/Linq/CacheLinqTest.Misc.cs | 2 +- .../Client/Cluster/ClientClusterGroupTests.cs | 1 - .../Compatibility/ClientServerCompatibilityTest.cs | 25 +- .../Common/IgniteProductVersionTests.cs | 33 +- .../Compute/ComputeApiTest.cs | 26 - .../Config/full-config.xml | 2 +- .../IgniteConfigurationSerializerTest.cs | 2 - .../IgniteConfigurationTest.cs | 2 - .../dotnet/Apache.Ignite.Core.Tests/JavaServer.cs | 18 +- .../dotnet/Apache.Ignite.Core.Tests/TestUtils.cs | 33 +- .../Apache.Ignite.Core.Tests/TestUtilsJni.cs | 19 + .../Client/IClientClusterNode.cs | 15 - .../Apache.Ignite.Core/Cluster/IClusterGroup.cs | 14 - .../Apache.Ignite.Core/Cluster/IClusterNode.cs | 15 - .../Common/IgniteProductVersion.cs | 12 +- .../Configuration/DataStorageConfiguration.cs | 2 +- .../Apache.Ignite.Core/IDataStorageMetrics.cs | 87 - .../platforms/dotnet/Apache.Ignite.Core/IIgnite.cs | 18 - .../Apache.Ignite.Core/IgniteConfiguration.cs | 21 - .../IgniteConfigurationSection.xsd | 8 - .../Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs | 20 - .../Impl/Client/Cluster/ClientClusterNode.cs | 11 +- .../Impl/Cluster/ClusterGroupImpl.cs | 37 - .../Impl/Cluster/ClusterNodeImpl.cs | 10 - .../Apache.Ignite.Core/Impl/DataStorageMetrics.cs | 87 - .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 25 +- .../Impl/PersistentStore/PersistentStoreMetrics.cs | 90 - .../Apache.Ignite.Core/Impl/Unmanaged/Jni/Env.cs | 16 + .../Impl/Unmanaged/Jni/EnvDelegates.cs | 6 +- .../Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs | 20 +- .../PersistentStore/IPersistentStoreMetrics.cs | 89 - .../PersistentStoreConfiguration.cs | 2 +- modules/platforms/dotnet/build.ps1 | 2 +- .../platforms/dotnet/examples/.vscode/launch.json | 82 +- .../dotnet/examples/Directory.Build.props | 6 +- modules/platforms/dotnet/examples/Shared/Utils.cs | 3 +- .../Apache.Ignite.Example/ExampleProject.csproj | 2 +- .../ExampleProjectThin.csproj | 2 +- .../Apache.Ignite.Examples.csproj | 2 +- .../protocols/http/jetty/GridJettyRestHandler.java | 1 - .../java/org/apache/ignite/IgniteSpringBean.java | 12 - .../IgniteProjectionStartStopRestartSelfTest.java | 107 +- modules/visor-console-2.10/README.txt | 4 - modules/visor-console-2.10/pom.xml | 180 -- modules/visor-console/licenses/apache-2.0.txt | 202 -- modules/visor-console/pom.xml | 170 -- .../ignite/visor/commands/VisorConsole.scala | 473 ---- .../visor/commands/ack/VisorAckCommand.scala | 143 - .../visor/commands/alert/VisorAlertCommand.scala | 938 ------- .../commands/cache/VisorCacheClearCommand.scala | 131 - .../visor/commands/cache/VisorCacheCommand.scala | 1032 -------- .../cache/VisorCacheLostPartitionsCommand.scala | 170 -- .../commands/cache/VisorCacheModifyCommand.scala | 413 --- .../cache/VisorCacheRebalanceCommand.scala | 134 - .../commands/cache/VisorCacheResetCommand.scala | 129 - .../VisorCacheResetLostPartitionsCommand.scala | 132 - .../commands/cache/VisorCacheScanCommand.scala | 237 -- .../commands/cache/VisorCacheStopCommand.scala | 136 - .../cache/VisorCacheToggleStatisticsCommand.scala | 152 -- .../commands/common/VisorConsoleCommand.scala | 104 - .../visor/commands/common/VisorTextTable.scala | 541 ---- .../config/VisorConfigurationCommand.scala | 461 ---- .../visor/commands/deploy/VisorDeployCommand.scala | 582 ----- .../commands/disco/VisorDiscoveryCommand.scala | 301 --- .../visor/commands/events/VisorEventsCommand.scala | 474 ---- .../ignite/visor/commands/gc/VisorGcCommand.scala | 196 -- .../visor/commands/kill/VisorKillCommand.scala | 431 --- .../visor/commands/node/VisorNodeCommand.scala | 379 --- .../visor/commands/open/VisorOpenCommand.scala | 307 --- .../visor/commands/ping/VisorPingCommand.scala | 217 -- .../visor/commands/start/VisorStartCommand.scala | 425 --- .../visor/commands/tasks/VisorTasksCommand.scala | 1445 ---------- .../visor/commands/top/VisorTopologyCommand.scala | 449 ---- .../visor/commands/vvm/VisorVvmCommand.scala | 301 --- .../main/scala/org/apache/ignite/visor/visor.scala | 2747 -------------------- .../apache/ignite/visor/VisorRuntimeBaseSpec.scala | 82 - .../apache/ignite/visor/VisorTextTableSpec.scala | 45 - .../ignite/visor/commands/VisorArgListSpec.scala | 72 - .../commands/VisorFileNameCompleterSpec.scala | 58 - .../visor/commands/ack/VisorAckCommandSpec.scala | 40 - .../commands/alert/VisorAlertCommandSpec.scala | 153 -- .../cache/VisorCacheClearCommandSpec.scala | 101 - .../commands/cache/VisorCacheCommandSpec.scala | 164 -- .../cache/VisorCacheResetCommandSpec.scala | 114 - .../config/VisorConfigurationCommandSpec.scala | 50 - .../commands/deploy/VisorDeployCommandSpec.scala | 34 - .../commands/disco/VisorDiscoveryCommandSpec.scala | 85 - .../commands/events/VisorEventsCommandSpec.scala | 62 - .../visor/commands/gc/VisorGcCommandSpec.scala | 61 - .../visor/commands/help/VisorHelpCommandSpec.scala | 73 - .../visor/commands/kill/VisorKillCommandSpec.scala | 59 - .../visor/commands/log/VisorLogCommandSpec.scala | 32 - .../commands/mem/VisorMemoryCommandSpec.scala | 115 - .../visor/commands/node/VisorNodeCommandSpec.scala | 42 - .../visor/commands/open/VisorOpenCommandSpec.scala | 36 - .../visor/commands/ping/VisorPingCommandSpec.scala | 38 - .../commands/start/VisorStartCommandSpec.scala | 124 - .../commands/tasks/VisorTasksCommandSpec.scala | 221 -- .../commands/top/VisorActivationCommandSpec.scala | 70 - .../commands/top/VisorTopologyCommandSpec.scala | 62 - .../visor/commands/vvm/VisorVvmCommandSpec.scala | 47 - .../testsuites/VisorConsoleSelfTestSuite.scala | 93 - modules/visor-plugins/licenses/apache-2.0.txt | 202 -- modules/visor-plugins/pom.xml | 86 - .../visor-plugins/src/main/java/log4j2.properties | 31 - .../ignite/visor/plugin/VisorExtensionPoint.java | 75 - .../ignite/visor/plugin/VisorPluggableTab.java | 64 - .../apache/ignite/visor/plugin/VisorPlugin.java | 53 - .../visor/plugin/VisorPluginComponentsFactory.java | 49 - .../ignite/visor/plugin/VisorPluginModel.java | 97 - .../ignite/visor/plugin/VisorPluginNodesPanel.java | 49 - .../ignite/visor/plugin/VisorTopologyListener.java | 53 - .../failover/IgniteFailoverAbstractBenchmark.java | 4 +- .../zk/internal/ZookeeperClusterNode.java | 20 - .../zk/internal/ZookeeperDiscoveryImpl.java | 2 +- .../zk/internal/ZookeeperDiscoverySpiTestBase.java | 4 + packaging/rpm/apache-ignite.spec | 5 - pom.xml | 67 - 712 files changed, 8302 insertions(+), 54607 deletions(-) diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 8c08e4dc1bb,964bbbf67f1..88fab6a355c --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@@ -18,7 -18,9 +18,8 @@@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; import java.io.BufferedInputStream; + import java.io.Closeable; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@@ -104,12 -109,9 +109,13 @@@ import org.apache.ignite.internal.manag import org.apache.ignite.internal.managers.encryption.GroupKeyEncrypted; import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener; import org.apache.ignite.internal.managers.systemview.walker.SnapshotViewWalker; + import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.RolloverType; +import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; import org.apache.ignite.internal.processors.cache.CacheObjectContext; import org.apache.ignite.internal.processors.cache.CacheType; @@@ -817,154 -805,6 +849,154 @@@ public class IgniteSnapshotManager exte "on the local node [missed=" + leftGrps + ", nodeId=" + cctx.localNodeId() + ']')); } + if (req.incremental()) { + SnapshotMetadata meta; + + try { + meta = readSnapshotMetadata(new File( + snapshotLocalDir(req.snapshotName(), req.snapshotPath()), + snapshotMetaFileName(cctx.localNode().consistentId().toString()) + )); + + checkIncrementalCanBeCreated(req.snapshotName(), req.snapshotPath(), meta); + } + catch (IgniteCheckedException | IOException e) { + return new GridFinishedFuture<>(e); + } + + return initLocalIncrementalSnapshot(req, meta); + } + else + return initLocalFullSnapshot(req, grpIds, withMetaStorage); + } + + /** + * @param req Request on snapshot creation. + * @param meta Full snapshot metadata. + * @return Future which will be completed when a snapshot has been started. + */ + private IgniteInternalFuture<SnapshotOperationResponse> initLocalIncrementalSnapshot( + SnapshotOperationRequest req, + SnapshotMetadata meta + ) { + File incSnpDir = incrementalSnapshotLocalDir(req.snapshotName(), req.snapshotPath(), req.incrementIndex()); + WALPointer lowPtr, highPtr; + + if (req.incrementIndex() == 1) + lowPtr = meta.snapshotRecordPointer(); + else { + int prevIdx = req.incrementIndex() - 1; + + IncrementalSnapshotMetadata prevIncSnpMeta; + + try { + prevIncSnpMeta = readFromFile(new File( + incrementalSnapshotLocalDir(req.snapshotName(), req.snapshotPath(), prevIdx), + incrementalSnapshotMetaFileName(prevIdx) + )); + } + catch (IgniteCheckedException | IOException e) { + return new GridFinishedFuture<>(e); + } + + lowPtr = prevIncSnpMeta.cutPointer(); + } + + cctx.database().checkpointReadLock(); + + try { + highPtr = cctx.wal().log(new ClusterSnapshotRecord(req.snapshotName())); + + // Dummy way to forcefully switch to the next segment. + // TODO: Must be replaced with the actual ConsistentCut logging. + cctx.wal().log(new ClusterSnapshotRecord(req.snapshotName()), RolloverType.NEXT_SEGMENT); + } + catch (IgniteCheckedException e) { + return new GridFinishedFuture<>(e); + } + finally { + cctx.database().checkpointReadUnlock(); + } + + // For now, forcefully rollover to the next WAL segments to make segments waiting possible. + assert cctx.wal().currentSegment() >= highPtr.index() : "Rollover must be invoked."; + + IgniteInternalFuture<SnapshotOperationResponse> task0 = registerTask(req.snapshotName(), new IncrementalSnapshotFutureTask( + cctx, + req.operationalNodeId(), + req.requestId(), + meta, + req.snapshotPath(), + req.incrementIndex(), + tmpWorkDir, + ioFactory, + lowPtr, + highPtr + )).chain(fut -> { + if (fut.error() != null) + throw F.wrap(fut.error()); + + assert incSnpDir.exists() : "Incremental snapshot directory must exists"; + + IncrementalSnapshotMetadata incMeta = new IncrementalSnapshotMetadata( + req.requestId(), + req.snapshotName(), + req.incrementIndex(), + cctx.localNode().consistentId().toString(), + pdsSettings.folderName(), + highPtr + ); + - writeSnapshotMetafile( - new File(incSnpDir, incrementalSnapshotMetaFileName(req.incrementIndex())), - incMeta ++ storeSnapshotMeta( ++ incMeta, ++ new File(incSnpDir, incrementalSnapshotMetaFileName(req.incrementIndex())) + ); + + return new SnapshotOperationResponse(); + }); + + if (task0.isDone()) + return task0; + + if (log.isDebugEnabled()) { + log.debug("Incremental snapshot operation submited for execution" + + "[snpName=" + req.snapshotName() + ", incIdx=" + req.incrementIndex()); + } + + clusterSnpReq = req; + + cctx.kernalContext().pools().getSnapshotExecutorService().submit(() -> { + SnapshotOperationRequest snpReq = clusterSnpReq; + + AbstractSnapshotFutureTask<?> task = locSnpTasks.get(snpReq.snapshotName()); + + if (task == null) + return; + + if (log.isDebugEnabled()) { + log.debug("Incremental snapshot operation started" + + "[snpName=" + req.snapshotName() + ", incIdx=" + req.incrementIndex()); + } + + writeSnapshotDirectoryToMetastorage(incSnpDir); + + task.start(); + }); + + return task0; + } + + /** + * @param req Request + * @param grpIds Groups. + * @param withMetaStorage Flag to include metastorage. + * @return Create snapshot future. + */ + private IgniteInternalFuture<SnapshotOperationResponse> initLocalFullSnapshot( + SnapshotOperationRequest req, + List<Integer> grpIds, + boolean withMetaStorage + ) { Map<Integer, Set<Integer>> parts = new HashMap<>(); // Prepare collection of pairs group and appropriate cache partition to be snapshot. @@@ -1023,47 -858,23 +1053,23 @@@ cctx.gridConfig().getEncryptionSpi().masterKeyDigest() ); - writeSnapshotMetafile( - new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString())), - meta - ); + SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, req.groups(), cctx.localNode(), snpDir, + req.streamerWarning()); + + req.meta(meta); - SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, req.groups(), cctx.localNode(), - snpDir, req.streamerWarning()); + File smf = new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString())); + - storeSnapshotMeta(req, smf); ++ storeSnapshotMeta(req.meta(), smf); + + log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); return new SnapshotOperationResponse(handlers.invokeAll(SnapshotHandlerType.CREATE, ctx)); } - catch (IOException | IgniteCheckedException e) { + catch (IgniteCheckedException e) { throw F.wrap(e); } - }); - } - - /** - * @param smf File to write to. - * @param meta Snapshot meta information. - */ - private <M extends Serializable> void writeSnapshotMetafile(File smf, M meta) { - if (smf.exists()) - throw new GridClosureException(new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath())); - - try (OutputStream out = Files.newOutputStream(smf.toPath())) { - byte[] bytes = U.marshal(marsh, meta); - int blockSize = SNAPSHOT_LIMITED_TRANSFER_BLOCK_SIZE_BYTES; - - for (int off = 0; off < bytes.length; off += blockSize) { - int len = Math.min(blockSize, bytes.length - off); - - transferRateLimiter.acquire(len); - - out.write(bytes, off, len); - } - } - catch (IOException | IgniteCheckedException e) { - throw new IgniteException(e); - } - - log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); + }, snapshotExecutorService()); } /** @@@ -1129,6 -940,31 +1135,33 @@@ } } + /** + * Stores snapshot metadata. + * - * @param snpReq Snapshot operation request containing snapshot meta. ++ * @param meta Metadata to store. + * @param smf File to store. + */ - private void storeSnapshotMeta(SnapshotOperationRequest snpReq, File smf) - throws IgniteCheckedException, IOException { ++ private <M extends Serializable> void storeSnapshotMeta(M meta, File smf) { + if (smf.exists()) + throw new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath()); + + try (OutputStream out = Files.newOutputStream(smf.toPath())) { - byte[] bytes = U.marshal(marsh, snpReq.meta()); ++ byte[] bytes = U.marshal(marsh, meta); + int blockSize = SNAPSHOT_LIMITED_TRANSFER_BLOCK_SIZE_BYTES; + + for (int off = 0; off < bytes.length; off += blockSize) { + int len = Math.min(blockSize, bytes.length - off); + + transferRateLimiter.acquire(len); + + out.write(bytes, off, len); + } + } ++ catch (IOException | IgniteCheckedException e) { ++ throw new IgniteException(e); ++ } + } + /** * Execute the {@link SnapshotHandler#complete(String, Collection)} method of the snapshot handlers asynchronously. * @@@ -1193,11 -1029,17 +1226,20 @@@ if (req.error() != null) { snpReq.error(req.error()); - deleteSnapshot(snapshotLocalDir(req.snapshotName(), req.snapshotPath()), pdsSettings.folderName()); + if (req.incremental()) + U.delete(incrementalSnapshotLocalDir(req.snapshotName(), req.snapshotPath(), req.incrementIndex())); + else + deleteSnapshot(snapshotLocalDir(req.snapshotName(), req.snapshotPath()), pdsSettings.folderName()); } + else if (!F.isEmpty(req.warnings())) { + // Pass the warnings further to the next stage for the case when snapshot started from not coordinator. + if (!isLocalNodeCoordinator(cctx.discovery())) + snpReq.warnings(req.warnings()); + + snpReq.meta().warnings(Collections.unmodifiableList(req.warnings())); + + storeWarnings(snpReq); + } removeLastMetaStorageKey(); } @@@ -1208,6 -1050,48 +1250,48 @@@ return new GridFinishedFuture<>(new SnapshotOperationResponse()); } + /** + * Stores snapshot creation warnings. The warnings are rare. Also, coordinator might not be a baseline node. Thus, + * storing meta with warnings once is to be done at second stage initialiation on any other node. Which leads to + * process possible snapshot errors, deleting snapshot at second stage end. Doesn't worth. If an error occurs on + * warnings writing, it is logged only. + */ + private void storeWarnings(SnapshotOperationRequest snpReq) { + assert !F.isEmpty(snpReq.warnings()); + + List<ClusterNode> snpNodes = cctx.kernalContext().cluster().get().nodes().stream() + .filter(n -> snpReq.nodes().contains(n.id())).collect(Collectors.toList()); + + boolean oldestBaseline = U.oldest(snpNodes, + n -> CU.baselineNode(n, cctx.kernalContext().state().clusterState())).equals(cctx.localNode()); + + if (!oldestBaseline) + return; + + File snpDir = snapshotLocalDir(snpReq.snapshotName(), snpReq.snapshotPath()); + File tempSmf = new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString()) + + SNAPSHOT_METAFILE_TMP_EXT); + File smf = new File(snpDir, snapshotMetaFileName(cctx.localNode().consistentId().toString())); + + try { - storeSnapshotMeta(snpReq, tempSmf); ++ storeSnapshotMeta(snpReq.meta(), tempSmf); + + Files.move(tempSmf.toPath(), smf.toPath(), StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + + if (log.isDebugEnabled()) + log.debug("Snapshot metafile has been rewrited with the warnings: " + smf.getAbsolutePath()); + } + catch (Exception e) { + log.error("Failed to store warnings of snapshot '" + snpReq.snapshotName() + + "' to the snapshot metafile. Snapshot won't contain them. The warnings: [" + + String.join(",", snpReq.warnings()) + "].", e); + } + finally { + U.delete(tempSmf); + } + } + /** * @param id Request id. * @param res Results. @@@ -1336,9 -1225,9 +1425,9 @@@ /** * @return List of all known snapshots on the local node. */ - public List<String> localSnapshotNames() { + public List<String> localSnapshotNames(@Nullable String snpPath) { if (cctx.kernalContext().clientNode()) - throw new UnsupportedOperationException("Client and daemon nodes can not perform this operation."); + throw new UnsupportedOperationException("Client nodes can not perform this operation."); if (locSnpDir == null) return Collections.emptyList(); diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java index 8200740a4a0,0a74d50cdb2..5c0a71b6243 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadata.java @@@ -84,9 -79,12 +84,13 @@@ public class SnapshotMetadata implement @GridToStringInclude @Nullable private final byte[] masterKeyDigest; + /** Warnings occurred at snapshot creation. */ + @GridToStringInclude + @Nullable private List<String> warnings; + /** - * F@param snpName Snapshot name. + * @param rqId Unique request id. + * @param snpName Snapshot name. * @param consId Consistent id of a node to which this metadata relates. * @param folderName Directory name which stores the data files. * @param pageSize Page size of stored snapshot data. diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 030c006c818,4750c59a6d8..f3bd3fc111c --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@@ -138,12 -138,11 +138,13 @@@ public class IgniteClusterSnapshotResto for (int i = 0; i < CACHE_KEYS_RANGE; i++) ignite.cache(DEFAULT_CACHE_NAME).put(i, i); - ignite.context().cache().context().snapshotMgr().createSnapshot(SNAPSHOT_NAME, snpDir.toString()).get(TIMEOUT); + ignite.context().cache().context().snapshotMgr() + .createSnapshot(SNAPSHOT_NAME, snpDir.toString(), false) + .get(TIMEOUT); // Check snapshot. - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir.getAbsolutePath()).get(TIMEOUT); + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME, snpDir.getAbsolutePath()).get(TIMEOUT) + .idleVerifyResult(); StringBuilder sb = new StringBuilder(); res.print(sb::append, true); diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java index 28b441df055,f0d6141dca8..ddb59e26f4c --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotStreamerTest.java @@@ -306,27 -415,56 +415,56 @@@ public class IgniteClusterSnapshotStrea } /** */ - private void createAndCheckSnapshot(String expWrn, String notexpWrn) throws IgniteCheckedException { - assert notexpWrn == null || expWrn != null; + private SnapshotPartitionsVerifyTaskResult createAndCheckSnapshot(IgniteEx snpHnd, boolean create, + String expWrn, String notExpWrn) throws Exception { + assert notExpWrn == null || expWrn != null; - if (expWrn == null) - snpMgr.createSnapshot(SNAPSHOT_NAME, null, false).get(); - else { - Throwable snpWrn = assertThrows( - null, - () -> snpMgr.createSnapshot(SNAPSHOT_NAME, null, false).get(), - IgniteException.class, - expWrn - ); - - if (notexpWrn != null) - assertTrue(!snpWrn.getMessage().contains(notexpWrn)); + if (create) { + if (expWrn == null) - snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null).get(); ++ snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null, false).get(); + else { + Throwable snpWrn = assertThrows( + null, - () -> snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null).get(), ++ () -> snp(snpHnd).createSnapshot(SNAPSHOT_NAME, null, false).get(), + IgniteException.class, + expWrn + ); + + if (notExpWrn != null) + assertTrue(!snpWrn.getMessage().contains(notExpWrn)); + } } - IdleVerifyResultV2 checkRes = snpMgr.checkSnapshot(SNAPSHOT_NAME, null).get(); + SnapshotPartitionsVerifyTaskResult checkRes = snp(snpHnd).checkSnapshot(SNAPSHOT_NAME, null).get(); assertTrue(checkRes.exceptions().isEmpty()); - assertTrue((expWrn != null) == checkRes.hasConflicts()); + assertTrue((expWrn != null) == checkRes.idleVerifyResult().hasConflicts()); + + if (expWrn != null) { + ListeningTestLogger testLog = new ListeningTestLogger(); + + LogListener lsnr = LogListener.matches(expWrn).times(1).build(); + + testLog.registerListener(lsnr); + + checkRes.print(testLog::info); + + lsnr.check(); + + if (notExpWrn != null) { + testLog = new ListeningTestLogger(); + + lsnr = LogListener.matches(notExpWrn).times(0).build(); + + testLog.registerListener(lsnr); + + checkRes.print(testLog::info); + + lsnr.check(); + } + } + + return checkRes; } /** */ diff --cc modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite.java index a0217b4779d,33419af614b..e49b0dc0aeb --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite.java @@@ -50,7 -51,8 +52,9 @@@ import org.junit.runners.Suite EncryptedSnapshotTest.class, IgniteClusterSnapshotWalRecordTest.class, IgniteClusterSnapshotStreamerTest.class, + IgniteSnapshotConsistencyTest.class, - IgniteClusterSnapshotDeltaTest.class ++ IgniteClusterSnapshotDeltaTest.class, + IncrementalSnapshotTest.class }) public class IgniteSnapshotTestSuite { }
