This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git
commit a5ef55f51fb987da7dac75e5fe1d56437e541826 Merge: 8c9bd4f 99fbecd Author: sboikov <[email protected]> AuthorDate: Tue Jul 23 06:49:33 2019 +0300 ignite-11704 bin/control.bat | 1 - bin/control.sh | 1 - bin/ignite-tf.sh | 1 - bin/ignite.bat | 1 - bin/ignite.sh | 1 - bin/ignitevisorcmd.bat | 1 - bin/ignitevisorcmd.sh | 1 - .../benchmarks/jmh/tree/BPlusTreeBenchmark.java | 4 +- .../benchmarks/jol/GridMetricsJolBenchmark.java | 12 +- .../thin/JdbcThinDataPageScanPropertySelfTest.java | 9 +- .../compatibility/PdsWithTtlCompatibilityTest.java | 197 ---------- .../IgniteCompatibilityBasicTestSuite.java | 3 - modules/compress/pom.xml | 15 + .../DiskPageCompressionConfigValidationTest.java | 161 ++++++++ .../DiskPageCompressionIntegrationTest.java | 19 +- .../testsuites/IgnitePdsCompressionTestSuite.java | 2 + modules/compress/src/test/resources/encryption.jks | Bin 0 -> 6737 bytes .../main/java/org/apache/ignite/IgniteCache.java | 35 ++ .../apache/ignite/IgniteJdbcThinDataSource.java | 17 - .../apache/ignite/cache/query/QueryMetrics.java | 6 +- .../org/apache/ignite/cache/query/ScanQuery.java | 31 -- .../apache/ignite/cache/query/SqlFieldsQuery.java | 32 -- .../org/apache/ignite/cache/query/SqlQuery.java | 31 -- .../ignite/client/ClientConnectionException.java | 18 +- .../events/CacheConsistencyViolationEvent.java | 69 ++++ .../java/org/apache/ignite/events/EventType.java | 15 + .../org/apache/ignite/internal/IgniteFeatures.java | 5 +- .../org/apache/ignite/internal/IgniteKernal.java | 26 +- .../org/apache/ignite/internal/IgnitionEx.java | 65 ++-- .../ignite/internal/client/thin/ClientChannel.java | 17 +- .../client/thin/ClientFieldsQueryPager.java | 7 +- .../internal/client/thin/ClientQueryPager.java | 7 +- .../ignite/internal/client/thin/ClientUtils.java | 11 +- .../internal/client/thin/GenericQueryPager.java | 42 +- .../thin/PayloadInputChannel.java} | 43 ++- .../internal/client/thin/PayloadOutputChannel.java | 62 +++ .../internal/client/thin/ProtocolVersion.java | 9 + .../internal/client/thin/ReliableChannel.java | 121 +++--- .../internal/client/thin/TcpClientCache.java | 98 +++-- .../internal/client/thin/TcpClientChannel.java | 276 ++++++++++--- .../internal/client/thin/TcpIgniteClient.java | 56 ++- .../ignite/internal/commandline/WalCommands.java | 13 +- .../internal/commandline/cache/CacheCommands.java | 3 +- .../ignite/internal/jdbc/JdbcConnection.java | 19 +- .../jdbc/JdbcConnectionValidationTask.java | 54 +++ .../internal/logger/platform/PlatformLogger.java | 2 +- .../managers/discovery/ClusterMetricsImpl.java | 30 +- .../IncompleteDeserializationException.java | 53 +++ .../eventstorage/GridEventStorageManager.java | 14 +- .../internal/metric/IoStatisticsHolderCache.java | 8 +- .../internal/metric/IoStatisticsHolderIndex.java | 12 +- .../pagemem/impl/PageMemoryNoStoreImpl.java | 20 +- .../pagemem/store/IgnitePageStoreManager.java | 4 +- .../internal/pagemem/wal/record/WALRecord.java | 166 ++++---- .../processors/cache/CacheGroupContext.java | 25 +- .../processors/cache/CacheGroupMetricsImpl.java | 428 ++++++++++++++++++++- .../cache/CacheGroupMetricsMXBeanImpl.java | 266 ++----------- .../processors/cache/CacheMetricsImpl.java | 138 ++++--- .../processors/cache/CacheOperationContext.java | 55 ++- .../cache/GatewayProtectedCacheProxy.java | 34 ++ .../processors/cache/GridCacheAdapter.java | 376 +++++++++++++++--- .../processors/cache/GridCachePreloader.java | 12 - .../cache/GridCachePreloaderAdapter.java | 14 - .../processors/cache/GridCacheProcessor.java | 108 ++++-- .../processors/cache/GridCacheProxyImpl.java | 54 ++- .../cache/IgniteCacheOffheapManagerImpl.java | 9 +- .../processors/cache/IgniteCacheProxyImpl.java | 7 +- .../internal/processors/cache/WalStateManager.java | 13 +- .../cache/distributed/dht/GridDhtCacheAdapter.java | 3 + .../distributed/dht/GridPartitionedGetFuture.java | 41 +- .../distributed/dht/atomic/GridDhtAtomicCache.java | 74 +++- .../dht/colocated/GridDhtColocatedCache.java | 38 +- .../dht/preloader/GridDhtPartitionDemander.java | 69 ++-- .../dht/preloader/GridDhtPartitionSupplier.java | 21 +- .../dht/preloader/GridDhtPreloader.java | 12 - .../distributed/near/GridNearAtomicCache.java | 3 +- .../near/GridNearTransactionalCache.java | 2 + .../cache/distributed/near/GridNearTxLocal.java | 121 +++++- .../GridNearReadRepairAbstractFuture.java | 221 +++++++++++ .../GridNearReadRepairCheckOnlyFuture.java | 176 +++++++++ .../near/consistency/GridNearReadRepairFuture.java | 166 ++++++++ .../IgniteConsistencyViolationException.java} | 21 +- .../cache/local/atomic/GridLocalAtomicCache.java | 12 +- .../processors/cache/mvcc/MvccProcessorImpl.java | 2 +- .../cache/persistence/CacheDataRowAdapter.java | 19 +- .../cache/persistence/DataRegionMetricsImpl.java | 133 +++---- .../cache/persistence/DataStorageMetricsImpl.java | 50 +-- .../GridCacheDatabaseSharedManager.java | 60 +-- .../IgniteCacheDatabaseSharedManager.java | 6 +- .../cache/persistence/file/FilePageStore.java | 16 +- .../persistence/file/FilePageStoreFactory.java | 6 +- .../persistence/file/FilePageStoreManager.java | 13 +- .../cache/persistence/file/FilePageStoreV2.java | 4 +- .../file/FileVersionCheckingFactory.java | 6 +- .../cache/persistence/pagemem/PageMemoryImpl.java | 13 +- .../cache/persistence/tree/BPlusTree.java | 13 +- .../wal/AbstractWalRecordsIterator.java | 15 +- .../persistence/wal/FileWriteAheadLogManager.java | 11 +- .../wal/serializer/RecordV1Serializer.java | 15 +- .../wal/serializer/RecordV2Serializer.java | 4 +- .../processors/cache/query/CacheQuery.java | 13 - .../cache/query/GridCacheQueryAdapter.java | 18 - .../cache/query/GridCacheQueryManager.java | 10 +- .../cache/query/GridCacheQueryMetricsAdapter.java | 206 ++++++---- .../cache/transactions/IgniteTxEntry.java | 2 +- .../IgniteTxImplicitSingleStateImpl.java | 5 + .../cache/transactions/IgniteTxLocalState.java | 5 + .../cache/transactions/IgniteTxManager.java | 10 + .../cache/transactions/IgniteTxStateImpl.java | 5 + .../processors/cache/tree/CacheDataTree.java | 8 +- .../cache/version/GridCacheVersionManager.java | 2 +- .../continuous/GridContinuousProcessor.java | 62 ++- .../continuous/StartRoutineDiscoveryMessage.java | 41 +- .../datastructures/GridCacheQueueAdapter.java | 1 + .../internal/processors/job/GridJobProcessor.java | 36 +- .../persistence/DistributedMetaStorageImpl.java | 52 ++- .../processors/metric/GridMetricManager.java | 38 +- .../internal/processors/metric/MetricRegistry.java | 37 +- .../{LongMetricImpl.java => AtomicLongMetric.java} | 8 +- ...ngAdderMetricImpl.java => LongAdderMetric.java} | 4 +- ...cImpl.java => LongAdderWithDelegateMetric.java} | 46 +-- .../processors/metric/impl/MetricUtils.java | 30 +- .../processors/odbc/jdbc/JdbcRequestHandler.java | 5 +- .../processors/query/GridQueryProcessor.java | 3 +- .../twostep/messages/GridQueryNextPageRequest.java | 2 - .../query/schema/SchemaIndexCacheVisitorImpl.java | 6 +- .../internal/util/future/GridCompoundFuture.java | 14 +- .../util/io/GridByteArrayOutputStream.java | 5 + .../util/tostring/GridToStringBuilder.java | 47 ++- .../util/tostring/GridToStringClassDescriptor.java | 12 +- .../util/tostring/GridToStringFieldDescriptor.java | 98 ++++- .../ignite/marshaller/jdk/JdkMarshaller.java | 43 +-- .../ignite/mxbean/CacheGroupMetricsMXBean.java | 15 +- .../ignite/spi/discovery/tcp/ServerImpl.java | 200 ++++++++-- .../ignite/spi/discovery/tcp/TcpDiscoverySpi.java | 5 +- .../messages/TcpDiscoveryCustomEventMessage.java | 14 +- .../messages/TcpDiscoveryDuplicateIdMessage.java | 26 ++ .../messages/TcpDiscoveryStatusCheckMessage.java | 33 +- .../org/apache/ignite/client/AsyncChannelTest.java | 198 ++++++++++ ...idEventStorageRuntimeConfigurationSelfTest.java | 10 +- .../commandline/CommandHandlerParsingTest.java | 44 ++- .../IncompleteDeserializationExceptionTest.java | 132 +++++++ .../internal/metric/AbstractExporterSpiTest.java | 8 +- .../internal/metric/IoStatisticsCacheSelfTest.java | 8 +- ...IoStatisticsMetricsLocalMXBeanImplSelfTest.java | 12 +- .../internal/metric/IoStatisticsSelfTest.java | 18 +- .../ignite/internal/metric/MetricsSelfTest.java | 36 +- .../pagemem/impl/PageMemoryNoLoadSelfTest.java | 4 +- .../pagemem/wal/record/WALRecordTest.java} | 47 +-- ...csMBeanTest.java => CacheGroupMetricsTest.java} | 125 +++--- .../cache/IgniteClusterActivateDeactivateTest.java | 52 +-- ...usterActivateDeactivateTestWithPersistence.java | 4 + ...eanTest.java => MvccCacheGroupMetricsTest.java} | 2 +- .../consistency/AbstractFullSetReadRepairTest.java | 242 ++++++++++++ .../cache/consistency/AbstractReadRepairTest.java | 336 ++++++++++++++++ .../cache/consistency/AtomicReadRepairTest.java | 191 +++++++++ .../ExplicitTransactionalReadRepairTest.java | 149 +++++++ .../ImplicitTransactionalReadRepairTest.java | 99 +++++ ...icatedExplicitTransactionalReadRepairTest.java} | 21 +- ...icatedImplicitTransactionalReadRepairTest.java} | 18 +- ...BackupExplicitTransactionalReadRepairTest.java} | 9 +- ...BackupImplicitTransactionalReadRepairTest.java} | 9 +- .../GridCacheDhtEvictionNearReadersSelfTest.java | 2 +- .../GridCacheRebalancingWithAsyncClearingTest.java | 12 +- ...tePdsRestartAfterFailedToWriteMetaPageTest.java | 201 ++++++++++ .../persistence/IgnitePdsTaskCancelingTest.java | 3 +- ...IgnitePdsCacheWalDisabledOnRebalancingTest.java | 32 +- .../db/IgnitePdsReserveWalSegmentsTest.java | 6 +- .../db/wal/WalDeletionArchiveAbstractTest.java | 5 +- .../IgniteAbstractWalIteratorInvalidCrcTest.java | 2 +- .../pagemem/BPlusTreePageMemoryImplTest.java | 4 +- .../BPlusTreeReuseListPageMemoryImplTest.java | 4 +- .../IgnitePageMemReplaceDelayedWriteUnitTest.java | 3 +- .../pagemem/IndexStoragePageMemoryImplTest.java | 4 +- .../persistence/pagemem/NoOpPageStoreManager.java | 4 +- .../pagemem/PageMemoryImplNoLoadTest.java | 4 +- .../persistence/pagemem/PageMemoryImplTest.java | 5 +- .../pagemem/PageMemoryNoStoreLeakTest.java | 4 +- .../cache/query/CacheDataPageScanQueryTest.java | 24 +- ...ueryRemoteFilterMissingInClassPathSelfTest.java | 148 +++++-- .../cluster/BaselineAutoAdjustInMemoryTest.java | 26 ++ .../processors/database/BPlusTreeSelfTest.java | 9 +- .../processors/database/CacheFreeListSelfTest.java | 18 +- .../database/DataRegionMetricsSelfTest.java | 27 +- .../processors/database/IndexStorageSelfTest.java | 4 +- .../jobmetrics/GridJobMetricsSelfTest.java | 32 +- .../junits/multijvm/IgniteCacheProcessProxy.java | 5 + .../ignite/testsuites/IgniteBasicTestSuite.java | 22 +- .../IgniteCacheDataStructuresSelfTestSuite.java | 19 +- .../testsuites/IgniteCacheMvccTestSuite7.java | 8 +- .../ignite/testsuites/IgniteCacheTestSuite7.java | 4 +- .../ignite/testsuites/IgnitePdsTestSuite4.java | 5 + .../GridInternalTaskUnusedWalSegmentsTest.java | 17 +- .../ignite/internal/managers/discovery/Wrapper.ser | Bin 0 -> 260 bytes .../processors/query/h2/IgniteH2Indexing.java | 20 +- .../processors/query/h2/QueryParameters.java | 5 +- .../internal/processors/query/h2/dml/DmlUtils.java | 2 +- .../view/SqlSystemViewCacheGroupsIOStatistics.java | 10 +- .../query/h2/twostep/GridMapQueryExecutor.java | 1 - .../visor/verify/ValidateIndexesClosure.java | 13 +- .../org/apache/ignite/client/ClientTestSuite.java | 3 +- .../cache/CacheAbstractQueryMetricsSelfTest.java | 12 +- ...st.java => CacheGroupMetricsWithIndexTest.java} | 62 ++- .../cache/IgniteErrorOnRebalanceTest.java | 2 + .../IgniteCacheReplicatedQuerySelfTest.java | 27 +- .../processors/cache/index/H2RowCacheSelfTest.java | 15 +- .../cache/index/SchemaExchangeSelfTest.java | 11 +- .../query/h2/IgniteSqlQueryMinMaxTest.java | 2 + .../processors/query/h2/QueryDataPageScanTest.java | 31 +- .../query/h2/database/InlineIndexHelperTest.java | 12 +- .../IgniteCacheWithIndexingTestSuite.java | 4 +- .../cpp/binary/include/ignite/binary/binary_enum.h | 204 ++++++++++ .../include/ignite/binary/binary_enum_entry.h | 107 ++++++ .../include/ignite/binary/binary_raw_reader.h | 23 ++ .../include/ignite/binary/binary_raw_writer.h | 21 + .../binary/include/ignite/binary/binary_reader.h | 23 ++ .../cpp/binary/include/ignite/binary/binary_type.h | 8 +- .../binary/include/ignite/binary/binary_writer.h | 22 ++ .../include/ignite/impl/binary/binary_common.h | 6 + .../ignite/impl/binary/binary_reader_impl.h | 84 +++- .../include/ignite/impl/binary/binary_utils.h | 82 ++-- .../ignite/impl/binary/binary_writer_impl.h | 69 +++- .../ignite/impl/interop/interop_input_stream.h | 4 +- .../platforms/cpp/binary/project/vs/binary.vcxproj | 2 + .../cpp/binary/project/vs/binary.vcxproj.filters | 6 + .../cpp/binary/src/binary/binary_raw_reader.cpp | 5 + .../cpp/binary/src/binary/binary_raw_writer.cpp | 5 + .../cpp/binary/src/binary/binary_reader.cpp | 5 + .../cpp/binary/src/binary/binary_writer.cpp | 5 + .../binary/src/impl/binary/binary_reader_impl.cpp | 85 ++-- .../cpp/binary/src/impl/binary/binary_utils.cpp | 43 ++- .../binary/src/impl/binary/binary_writer_impl.cpp | 21 + .../core-test/include/ignite/binary_test_defs.h | 58 +++ .../cpp/core-test/src/binary_object_test.cpp | 117 ++++++ modules/platforms/cpp/core-test/src/cache_test.cpp | 50 ++- .../ExpiryCacheHolderTest.cs | 7 + .../Cache/CacheTestAsyncWrapper.cs | 12 + .../dotnet/Apache.Ignite.Core/Cache/ICache.cs | 13 + .../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 34 +- .../Apache.Ignite.Core/Impl/Cache/CacheOp.cs | 3 +- .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 2 +- modules/sqlline/bin/sqlline.bat | 1 - modules/sqlline/bin/sqlline.sh | 1 - .../web-console/web-agent/bin/ignite-web-agent.bat | 1 - .../web-console/web-agent/bin/ignite-web-agent.sh | 1 - parent/pom.xml | 3 +- 246 files changed, 7361 insertions(+), 2439 deletions(-) diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java index 5808db6,f26bf14..e002037 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java @@@ -29,8 -29,7 +29,8 @@@ import org.apache.ignite.internal.proce import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore; import org.apache.ignite.internal.processors.metric.MetricRegistry; import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; - import org.apache.ignite.internal.processors.metric.impl.LongAdderMetricImpl; - import org.apache.ignite.internal.processors.metric.impl.LongMetricImpl; ++import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; + import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@@ -160,11 -159,8 +160,11 @@@ public class CacheMetricsImpl implement private final HitRateMetric rebalancingBytesRate; /** Number of currently clearing partitions for rebalancing. */ - private final LongMetricImpl rebalanceClearingPartitions; + private final AtomicLongMetric rebalanceClearingPartitions; + /** */ - private LongAdderMetricImpl tombstones; ++ private LongAdderMetric tombstones; + /** Cache metrics. */ @GridToStringExclude private transient CacheMetricsImpl delegate; @@@ -314,10 -308,8 +312,10 @@@ REBALANCE_RATE_INTERVAL, 20); - rebalanceClearingPartitions = mreg.metric("RebalanceClearingPartitionsLeft", + rebalanceClearingPartitions = mreg.longMetric("RebalanceClearingPartitionsLeft", "Number of partitions need to be cleared before actual rebalance start."); + + tombstones = mreg.longAdderMetric("Tombstones", "Number of tombstone entries"); } /** diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index df46885,06e7214..f168903 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@@ -52,6 -52,6 +52,7 @@@ import static org.apache.ignite.interna import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.MVCC_OP_COUNTER_NA; import static org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.RowData.KEY_ONLY; import static org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.RowData.LINK_WITH_HEADER; ++import static org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.RowData.TOMBSTONES; /** * Cache data row adapter. @@@ -261,9 -261,9 +262,23 @@@ public class CacheDataRowAdapter implem incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); -- if (incomplete == null || (rowData == KEY_ONLY && key != null)) ++ if (incomplete == null) return; ++ if (rowData == KEY_ONLY) { ++ if (key != null) ++ return; ++ } ++ else if (rowData == TOMBSTONES) { ++ // TODO IGNITE-11704. ++ if (val != null && !sharedCtx.database().isTombstone(this)) { ++ verReady = true; ++ ver = null; ++ ++ return; ++ } ++ } ++ nextLink = incomplete.getNextLink(); } finally { @@@ -498,14 -498,6 +513,14 @@@ int len = PageUtils.getInt(addr, off); off += 4; + boolean tombstones = rowData == RowData.TOMBSTONES; + + if (tombstones && !sharedCtx.database().isTombstone(addr + off + len + 1)) { - verReady = true; ++ verReady = true; // Mark as ready, no need to read any data. + + return; + } + if (rowData != RowData.NO_KEY && rowData != RowData.NO_KEY_WITH_HINTS) { byte type = PageUtils.getByte(addr, off); off++;
