[ignite] branch master updated: IGNITE-10761 GridCacheProcessor should add info about cache in excecption message, if applicable. - Fixes #6697.
This is an automated email from the ASF dual-hosted git repository. irakov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 76b4a11 IGNITE-10761 GridCacheProcessor should add info about cache in excecption message, if applicable. - Fixes #6697. 76b4a11 is described below commit 76b4a112ce4c035db83ab43b7e2f66973d15eb62 Author: Kirill Tkalenko AuthorDate: Tue Jul 23 22:16:00 2019 +0300 IGNITE-10761 GridCacheProcessor should add info about cache in excecption message, if applicable. - Fixes #6697. Signed-off-by: Ivan Rakov --- .../processors/cache/GridCacheProcessor.java | 63 ++ 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index baf4c84..daa2992 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -36,6 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.cache.configuration.FactoryBuilder; @@ -241,6 +242,12 @@ public class GridCacheProcessor extends GridProcessorAdapter { "(encryption settings are different for cache '%s' : local=%s, remote=%s.)"; /** */ +private static final String CACHE_NAME_AND_OPERATION_FORMAT = "[cacheName=%s, operation=%s]"; + +/** */ +private static final String CACHE_NAMES_AND_OPERATION_FORMAT = "[cacheNames=%s, operation=%s]"; + +/** */ private final boolean startClientCaches = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_START_CACHES_ON_JOIN, false); @@ -4056,8 +4063,10 @@ public class GridCacheProcessor extends GridProcessorAdapter { ) { assert cacheName != null; -if (checkThreadTx) -checkEmptyTransactions(); +if (checkThreadTx) { +checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, cacheName, +"dynamicStartCache")); +} GridPlainClosure, IgniteInternalFuture> startCacheClsr = (grpKeys) -> { assert ccfg == null || !ccfg.isEncryptionEnabled() || !grpKeys.isEmpty(); @@ -4206,8 +4215,16 @@ public class GridCacheProcessor extends GridProcessorAdapter { boolean disabledAfterStart, IgniteUuid restartId ) { -if (checkThreadTx) -checkEmptyTransactions(); +if (checkThreadTx) { +checkEmptyTransactionsEx(() -> { +List cacheNames = storedCacheDataList.stream() +.map(StoredCacheData::config) +.map(CacheConfiguration::getName) +.collect(Collectors.toList()); + +return String.format(CACHE_NAMES_AND_OPERATION_FORMAT, cacheNames, "dynamicStartCachesByStoredConf"); +}); +} GridPlainClosure, IgniteInternalFuture> startCacheClsr = (grpKeys) -> { List srvReqs = null; @@ -4309,8 +4326,10 @@ public class GridCacheProcessor extends GridProcessorAdapter { ) { assert cacheName != null; -if (checkThreadTx) -checkEmptyTransactions(); +if (checkThreadTx) { +checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, cacheName, +"dynamicDestroyCache")); +} DynamicCacheChangeRequest req = DynamicCacheChangeRequest.stopRequest(ctx, cacheName, sql, true); @@ -4342,8 +4361,10 @@ public class GridCacheProcessor extends GridProcessorAdapter { boolean checkThreadTx, boolean destroy ) { -if (checkThreadTx) -checkEmptyTransactions(); +if (checkThreadTx) { +checkEmptyTransactionsEx(() -> String.format(CACHE_NAMES_AND_OPERATION_FORMAT, cacheNames, +"dynamicDestroyCaches")); +} List reqs = new ArrayList<>(cacheNames.size()); @@ -4429,7 +4450,7 @@ public class GridCacheProcessor extends GridProcessorAdapter { if (proxy == null || proxy.isProxyClosed()) return new GridFinishedFuture<>(); // No-op. -checkEmptyTransactions(); +checkEmptyTransactionsEx(() -> String.format(CACHE_NAME_AND_OPERATION_FORMAT, cacheName, "dynamicCloseCache")); if (proxy.context().isLocal()) return dynamicDestroyCache(cacheName, false, true, false, null); @@ -,11 +4465,14 @@ public
[ignite] branch master updated: IGNITE-11993 Print warning if awaiting next wal segment it too long - Fixes #6703.
This is an automated email from the ASF dual-hosted git repository. dgovorukhin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new a736544 IGNITE-11993 Print warning if awaiting next wal segment it too long - Fixes #6703. a736544 is described below commit a7365445e53d24aa80a54127b5407cdbd30c20f8 Author: Kirill Tkalenko AuthorDate: Tue Jul 23 13:27:34 2019 +0300 IGNITE-11993 Print warning if awaiting next wal segment it too long - Fixes #6703. Signed-off-by: Dmitriy Govorukhin --- .../org/apache/ignite/IgniteSystemProperties.java | 7 +++ .../persistence/wal/FileWriteAheadLogManager.java | 22 ++ 2 files changed, 29 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 66acae0..9d785d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -914,6 +914,13 @@ public final class IgniteSystemProperties { public static final String IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE = "IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE"; /** + * Threshold time (in millis) to print warning to log if waiting for next wal segment took longer than the threshold. + * + * Default value is 1000 ms. + */ +public static final String IGNITE_THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT = "IGNITE_THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT"; + +/** * Count of WAL compressor worker threads. Default value is 4. */ public static final String IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT = "IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT"; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 9f2c352..d337944 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -135,6 +135,7 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP; @@ -234,6 +235,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private final int WAL_COMPRESSOR_WORKER_THREAD_CNT = IgniteSystemProperties.getInteger(IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT, 4); +/** + * Threshold time to print warning to log if awaiting for next wal segment took too long (exceeded this threshold). + */ +private static final long THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT = + IgniteSystemProperties.getLong(IGNITE_THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT, 1000L); + /** */ private final boolean alwaysWriteFullPages; @@ -1565,9 +1572,24 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl return new File(walWorkDir, FileDescriptor.fileName(curIdx + 1)); } +long absNextIdxStartTime = System.nanoTime(); + // Signal to archiver that we are done with the segment and it can be archived. long absNextIdx = archiver0.nextAbsoluteSegmentIndex(); +long absNextIdxWaitTime = U.nanosToMillis(System.nanoTime() - absNextIdxStartTime); + +if (absNextIdxWaitTime > THRESHOLD_WAIT_TIME_NEXT_WAL_SEGMENT) { +log.warning( +String.format("Waiting for next wal segment was too long " + +"[waitingTime=%s, curIdx=%s, absNextIdx=%s, walSegments=%s]", +absNextIdxWaitTime, +curIdx, +absNextIdx, +dsCfg.getWalSegments()) +); +} + long segmentIdx = absNextIdx % dsCfg.getWalSegments(); return new File(walWorkDir, FileDescriptor.fileName(segmentIdx));
[ignite] branch master updated: IGNITE-11990 Optimize heap usage for TcpDiscoveryNodeAddedMessage stored in pending messages in ServerImpl - Fixes #6700.
This is an automated email from the ASF dual-hosted git repository. dgovorukhin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 7847168 IGNITE-11990 Optimize heap usage for TcpDiscoveryNodeAddedMessage stored in pending messages in ServerImpl - Fixes #6700. 7847168 is described below commit 7847168836d1279941e1514331a1f29738cf6dae Author: vmalin AuthorDate: Tue Jul 23 13:24:10 2019 +0300 IGNITE-11990 Optimize heap usage for TcpDiscoveryNodeAddedMessage stored in pending messages in ServerImpl - Fixes #6700. Signed-off-by: Dmitriy Govorukhin --- .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java | 1 + .../ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java | 7 +++ .../spi/discovery/tcp/messages/TcpDiscoveryNodeAddedMessage.java | 9 + 3 files changed, 17 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 7ad4468..658cbf5 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -1807,6 +1807,7 @@ class ServerImpl extends TcpDiscoveryImpl { nodeAddedMsg.topology(null); nodeAddedMsg.topologyHistory(null); nodeAddedMsg.messages(null, null, null); +nodeAddedMsg.clearUnmarshalledDiscoveryData(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java index 09410aa..8ba19e1 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java @@ -477,4 +477,11 @@ public class DiscoveryDataPacket implements Serializable { public void joiningNodeClient(boolean joiningNodeClient) { this.joiningNodeClient = joiningNodeClient; } + +/** + * Clears {@link #unmarshalledJoiningNodeData} + */ +public void clearUnmarshalledJoiningNodeData() { +unmarshalledJoiningNodeData = null; +} } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeAddedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeAddedMessage.java index d2a0808..c1d8c89 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeAddedMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeAddedMessage.java @@ -228,6 +228,15 @@ public class TcpDiscoveryNodeAddedMessage extends TcpDiscoveryAbstractMessage { } /** + * Clears unmarshalled discovery data to minimize message size. + * These data are used only on "collect" stage and are not part of persistent state. + */ +public void clearUnmarshalledDiscoveryData() { +if (dataPacket != null) +dataPacket.clearUnmarshalledJoiningNodeData(); +} + +/** * @return First grid node start time. */ public long gridStartTime() {
[ignite] branch master updated: IGNITE-11995 control.sh if experimental command disabled - don't show help for experemental commands - Fixes #6704.
This is an automated email from the ASF dual-hosted git repository. dgovorukhin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new ac3714e IGNITE-11995 control.sh if experimental command disabled - don't show help for experemental commands - Fixes #6704. ac3714e is described below commit ac3714ef2bce71f0016e4535e9fef44c31b537a7 Author: Kirill Tkalenko AuthorDate: Tue Jul 23 11:27:15 2019 +0300 IGNITE-11995 control.sh if experimental command disabled - don't show help for experemental commands - Fixes #6704. Signed-off-by: Dmitriy Govorukhin --- bin/control.bat| 5 +++ .../ignite/internal/commandline/WalCommands.java | 48 -- .../commandline/CommandHandlerParsingTest.java | 30 ++ .../apache/ignite/util/GridCommandHandlerTest.java | 44 4 files changed, 80 insertions(+), 47 deletions(-) diff --git a/bin/control.bat b/bin/control.bat index c88b8bc..77a3f3f 100644 --- a/bin/control.bat +++ b/bin/control.bat @@ -170,6 +170,11 @@ if %ERRORLEVEL% equ 0 ( ) :: +:: Uncomment to enable experimental commands [--wal] +:: +:: set JVM_OPTS=%JVM_OPTS% -DIGNITE_ENABLE_EXPERIMENTAL_COMMAND=true + +:: :: Uncomment the following GC settings if you see spikes in your throughput due to Garbage Collection. :: :: set JVM_OPTS=%JVM_OPTS% -XX:+UseG1GC diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java index 8435a15..2960b36 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/WalCommands.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.visor.misc.VisorWalTaskResult; import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND; import static org.apache.ignite.internal.commandline.CommandArgIterator.isCommandOrOption; +import static org.apache.ignite.internal.commandline.CommandHandler.UTILITY_NAME; import static org.apache.ignite.internal.commandline.CommandList.WAL; import static org.apache.ignite.internal.commandline.CommandLogger.DOUBLE_INDENT; import static org.apache.ignite.internal.commandline.CommandLogger.INDENT; @@ -68,12 +69,13 @@ public class WalCommands implements Command> { /** {@inheritDoc} */ @Override public void printUsage(Logger logger) { -if (enableExperimental()) { -Command.usage(logger, "Print absolute paths of unused archived wal segments on each node:", WAL, -WAL_PRINT, "[consistentId1,consistentId2,,consistentIdN]"); -Command.usage(logger, "Delete unused archived wal segments on each node:", WAL, WAL_DELETE, -"[consistentId1,consistentId2,,consistentIdN]", optional(CMD_AUTO_CONFIRMATION)); -} +if (!enableExperimental()) +return; + +Command.usage(logger, "Print absolute paths of unused archived wal segments on each node:", WAL, +WAL_PRINT, "[consistentId1,consistentId2,,consistentIdN]"); +Command.usage(logger, "Delete unused archived wal segments on each node:", WAL, WAL_DELETE, +"[consistentId1,consistentId2,,consistentIdN]", optional(CMD_AUTO_CONFIRMATION)); } /** @@ -83,21 +85,26 @@ public class WalCommands implements Command> { * @throws Exception If failed to execute wal action. */ @Override public Object execute(GridClientConfiguration clientCfg, Logger logger) throws Exception { -this.logger = logger; +if (enableExperimental()) { +this.logger = logger; -try (GridClient client = Command.startClient(clientCfg)) { -switch (walAct) { -case WAL_DELETE: -deleteUnusedWalSegments(client, walArgs, clientCfg); +try (GridClient client = Command.startClient(clientCfg)) { +switch (walAct) { +case WAL_DELETE: +deleteUnusedWalSegments(client, walArgs, clientCfg); -break; +break; -case WAL_PRINT: -default: -printUnusedWalSegments(client, walArgs, clientCfg); +case WAL_PRINT: +default: +printUnusedWalSegments(client, walArgs, clientCfg); -break; +break; +} } +} else { +logger.warning(String.format("For use experimental command add %s=true to JVM_OPTS in %s", +IGNITE_ENABLE_EXPERIMENTAL_COMMAND, UTILITY_NAME)); } return null; @@ -113,9
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 0a574a6 ignite-11704 0a574a6 is described below commit 0a574a68385cf250b928972654b0486e2efe67a4 Author: sboikov AuthorDate: Tue Jul 23 10:42:28 2019 +0300 ignite-11704 --- .../cache/persistence/CacheDataRowAdapter.java | 13 +- .../CacheRemoveWithTombstonesLoadTest.java | 347 + .../ignite/testsuites/IgniteCacheTestSuite9.java | 2 + 3 files changed, 360 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 1be2bb5..5216e21 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -260,10 +260,19 @@ public class CacheDataRowAdapter implements CacheDataRow { int itemId = itemId(nextLink); incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, -grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); +grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); + +if (incomplete == null) { +if (rowData == TOMBSTONES && val != null && !sharedCtx.database().isTombstone(this)) { +// TODO IGNITE-11704. +ver = null; +key = null; +val = null; +verReady = true; +} -if (incomplete == null) return; +} if (rowData == KEY_ONLY) { if (key != null) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java new file mode 100644 index 000..2780132 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; +import static