[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r519811449
##
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java
##
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests;
+
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Loading random uuids to cache.
+ */
+public class UuidDataStreamerApplication extends IgniteAwareApplication {
+/** {@inheritDoc} */
+@Override public void run(JsonNode jNode) throws InterruptedException {
+String cacheName = jNode.get("cacheName").asText();
+
+int dataSize = Optional.ofNullable(jNode.get("dataSize"))
+.map(JsonNode::asInt)
+.orElse(1024);
+
+long iterSize = Optional.ofNullable(jNode.get("iterSize"))
+.map(JsonNode::asLong)
+.orElse(1024L);
+
+assert dataSize > 0;
+assert iterSize > 0;
+
+CacheConfiguration cacheCfg = new
CacheConfiguration<>(cacheName);
+cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+cacheCfg.setBackups(2);
+cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+cacheCfg.setIndexedTypes(UUID.class, byte[].class);
+
+ignite.getOrCreateCache(cacheCfg);
+
+long start = System.currentTimeMillis();
+
+markInitialized();
+
+workParallel(ignite, cacheName, iterSize, dataSize);
+
+recordResult("DURATION", System.currentTimeMillis() - start);
+
+markFinished();
+}
+
+/** */
+private void workParallel(Ignite ignite, String cacheName, long iterSize,
int dataSize)
+throws InterruptedException {
+int threads = Runtime.getRuntime().availableProcessors() / 2;
+
+long iterThread = iterSize / threads;
+
+assert iterThread > 0;
+
+CountDownLatch latch = new CountDownLatch(threads);
+
+ThreadFactory threadFactory = new ThreadFactoryBuilder()
+.setNameFormat("UuidDataStreamer-%d")
+.build();
+
+for (int i = 0; i < threads; i++)
+threadFactory.newThread(new UuidDataStreamer(ignite, cacheName,
latch, iterThread, dataSize))
+.start();
+
+latch.await();
+}
+
+/** */
+private static class UuidDataStreamer implements Runnable {
+/** Ignite. */
+private final Ignite ignite;
+
+/** Cache name. */
+private final String cacheName;
+
+/** Latch. */
+private final CountDownLatch latch;
+
+/** Iteration size. */
+private final long iterSize;
+
+/** Data size. */
+private final int dataSize;
+
+/** */
+public UuidDataStreamer(Ignite ignite, String cacheName,
CountDownLatch latch, long iterSize, int dataSize) {
+this.ignite = ignite;
+this.cacheName = cacheName;
+this.latch = latch;
+this.iterSize = iterSize;
+this.dataSize = dataSize;
+}
+
+/** {@inheritDoc} */
+@Override public void run() {
+try (IgniteDataStreamer dataStreamer =
ignite.dataStreamer(cacheName)) {
+dataStreamer.autoFlushFrequency(100L);
+
+for (long i = 0L; i <= iterSize; i++) {
+UUID uuid = UUID.randomUUID();
+
+byte[] data = new byte[dataSize];
+
+ThreadLocalRandom.current().nextBytes(data);
+
+dataStreamer.addData(uuid, data);
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r519812385
##
File path: modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
##
@@ -102,12 +102,13 @@ class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware):
"""
@property
def command(self):
-cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \
+cmd = "%s %s %s %s 2>&1 | tee %s >> %s &" % \
(self._envs(),
self.path.script("ignite.sh"),
self._jvm_opts(),
self.CONFIG_FILE,
- self.STDOUT_STDERR_CAPTURE)
+ self.STDOUT_STDERR_CAPTURE,
+ self.CONSOLE_ALL_CAPTURE)
Review comment:
in the chat discussion, we decided to do so
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294: URL: https://github.com/apache/ignite/pull/8294#discussion_r519812303 ## File path: modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py ## @@ -28,11 +28,13 @@ class PersistenceAware: """ # Root directory for persistent output PERSISTENT_ROOT = "/mnt/service" -STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") +PATH_TO_LOGS_DIR = os.path.join(PERSISTENT_ROOT, "logs") +STDOUT_STDERR_CAPTURE = os.path.join(PATH_TO_LOGS_DIR, "console.log") +CONSOLE_ALL_CAPTURE = os.path.join(PATH_TO_LOGS_DIR, "console_all.log") Review comment: in the chat discussion, we decided to do so This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r519812709
##
File path: modules/ducktests/tests/ignitetest/services/ignite.py
##
@@ -127,6 +141,36 @@ def clean_node(self, node):
node.account.kill_java_processes(self.APP_SERVICE_CLASS,
clean_shutdown=False, allow_fail=True)
node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT,
allow_fail=False)
+def rename_db(self, new_db_name: str):
+"""
+Rename db.
+"""
+for node in self.nodes:
+self._rename_db(node, new_db_name)
+
+def _rename_db(self, node, new_db_name: str):
+"""
+Rename db.
+"""
+assert len(self.pids(node)) == 0
+
+node.account.ssh(f"mv {self.WORK_DIR}/db
{self.WORK_DIR}/{new_db_name}")
+
+def restore_from_snapshot(self, snapshot_name: str):
+"""
+Copy from snapshot to db.
+"""
+for node in self.nodes:
+self._copy_snap_to_db(node, snapshot_name)
+
+def _copy_snap_to_db(self, node, snapshot_name: str):
Review comment:
fixed
##
File path: modules/ducktests/tests/ignitetest/services/ignite.py
##
@@ -127,6 +141,36 @@ def clean_node(self, node):
node.account.kill_java_processes(self.APP_SERVICE_CLASS,
clean_shutdown=False, allow_fail=True)
node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT,
allow_fail=False)
+def rename_db(self, new_db_name: str):
Review comment:
fixed
##
File path: modules/ducktests/tests/ignitetest/services/ignite.py
##
@@ -127,6 +141,36 @@ def clean_node(self, node):
node.account.kill_java_processes(self.APP_SERVICE_CLASS,
clean_shutdown=False, allow_fail=True)
node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT,
allow_fail=False)
+def rename_db(self, new_db_name: str):
+"""
+Rename db.
+"""
+for node in self.nodes:
+self._rename_db(node, new_db_name)
+
+def _rename_db(self, node, new_db_name: str):
Review comment:
fixed
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294: URL: https://github.com/apache/ignite/pull/8294#discussion_r519812504 ## File path: modules/ducktests/tests/ignitetest/services/ignite.py ## @@ -52,6 +52,20 @@ def start(self, timeout_sec=180): for node in self.nodes: self.await_node_started(node, timeout_sec) +def restart(self, timeout_sec=180): Review comment: fixed This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] a-polyakov opened a new pull request #8441: IGNITE-13687 Improvement of human-readable format of WAL records (StandaloneWalRecordsIterator)
a-polyakov opened a new pull request #8441: URL: https://github.com/apache/ignite/pull/8441 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] alamar opened a new pull request #8442: IGNITE-13665 When system worker is blocked, output its stack trace.
alamar opened a new pull request #8442: URL: https://github.com/apache/ignite/pull/8442 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite-extensions] asfgit closed pull request #26: IGNITE-13519 Adds Spring Data thin client support.
asfgit closed pull request #26: URL: https://github.com/apache/ignite-extensions/pull/26 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ptupitsyn opened a new pull request #8443: IGNITE-13357 .NET: Add IncludeExpired to ContinuousQuery and ContinuousQueryClient
ptupitsyn opened a new pull request #8443: URL: https://github.com/apache/ignite/pull/8443 * Add `IncludeExpired` flag to thin and thick continuous query APIs: `ContinuousQuery`, `ContinuousQueryClient` * Thin client protocol not affected: the flag is already supported This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] dmagda commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
dmagda commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r519979370 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: I guess the SSL issue is not the only reason of why developers might want to tweak this flag. How about this description? It's enough to mention the SSL issue on the SSL page only and leave the description in this place generic. _Specifies a linger-on-close timeout. This option disables/enables immediate return from a close() of a TCP Socket. Setting the timeout to `0` closes the socket immediately. See the Javadoc for more details._ ## File path: docs/_docs/security/ssl-tls.adoc ## @@ -32,6 +32,12 @@ To enable SSL/TLS for cluster nodes, configure an `SSLContext` factory in the no You can use the `org.apache.ignite.ssl.SslContextFactory`, which is the default factory that uses a configurable keystore to initialize the SSL context. //You can also implement your own `SSLContext` factory. +[NOTE] + Review comment: I would put it this way: _Consider setting the `TcpDiscovery.soLinger` parameter to X to avoid SSL-related deadlocks on certain versions of JRE. Alternatively, update your JRE version to the latest one._ And do the following with this text: 1. Turn the "SSL-related deadlock" phrase into an external link: https://github.com/apache/ignite/tree/master/docs#links-to-external-resources 2. The callout needs to be of the "caution" type: https://github.com/apache/ignite/tree/master/docs#callouts This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] dmagda commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
dmagda commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r519986691 ## File path: docs/_docs/security/ssl-tls.adoc ## @@ -32,6 +32,12 @@ To enable SSL/TLS for cluster nodes, configure an `SSLContext` factory in the no You can use the `org.apache.ignite.ssl.SslContextFactory`, which is the default factory that uses a configurable keystore to initialize the SSL context. //You can also implement your own `SSLContext` factory. +[NOTE] + Review comment: I would put it this way: _Consider setting the `TcpDiscovery.soLinger` parameter to X to avoid SSL-related deadlocks on certain versions of JRE. Alternatively, update your JRE version to the latest one._ And do the following with this text: 1. Turn the "SSL-related deadlock" phrase into an external link to the JDK ticket: https://github.com/apache/ignite/tree/master/docs#links-to-external-resources 2. The callout needs to be of the "caution" type: https://github.com/apache/ignite/tree/master/docs#callouts This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite-extensions] dmagda commented on a change in pull request #30: IGNITE-12951 Update documents for migrated extensions
dmagda commented on a change in pull request #30:
URL: https://github.com/apache/ignite-extensions/pull/30#discussion_r519992703
##
File path: modules/flink-ext/README.txt
##
@@ -20,7 +20,7 @@ interested in):
...
org.apache.ignite
-ignite-flink
+ignite-flink-ext
${ignite.version}
Review comment:
Should we replace the _ignite.version_ parameter with _ignite-flink-ext_
one as long as the version of an extension is different from the Ignite one.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8429: IGNITE-13678 Extend test coverage for persistence files directory and WAL page snapshot records compression
asfgit closed pull request #8429: URL: https://github.com/apache/ignite/pull/8429 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] akorensh commented on a change in pull request #8417: IGNITE-13655 Implement readiness probe REST endpoint
akorensh commented on a change in pull request #8417:
URL: https://github.com/apache/ignite/pull/8417#discussion_r520127297
##
File path: modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
##
@@ -1476,6 +1476,11 @@ public static DependencyResolver dependencyResolver() {
return dependencyResolver.get();
}
+public static boolean hasKernalStarted(String name) {
Review comment:
done
##
File path: modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
##
@@ -3209,6 +3214,13 @@ public void setCounter(int cnt) {
this.cnt = cnt;
}
}
+
+/**
+ *
+ * @return whether the startLatch has been counted down, thereby
indicating that the kernal has full started.
Review comment:
yes
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite-extensions] samaitra commented on a change in pull request #30: IGNITE-12951 Update documents for migrated extensions
samaitra commented on a change in pull request #30:
URL: https://github.com/apache/ignite-extensions/pull/30#discussion_r520205193
##
File path: modules/flink-ext/README.txt
##
@@ -20,7 +20,7 @@ interested in):
...
org.apache.ignite
-ignite-flink
+ignite-flink-ext
${ignite.version}
Review comment:
@dmagda thank you for reviewing the changes, I have updated the PR.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] a-polyakov opened a new pull request #8444: IGNITE-13689 Extend test coverage [IGNITE-11512] Add counter left partition for index rebuild in CacheGroupMetricsMXBean
a-polyakov opened a new pull request #8444: URL: https://github.com/apache/ignite/pull/8444 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Wuchte commented on a change in pull request #8358: IGNITE-12489 Fixed race between B+Tree cursor and remove operation.
Wuchte commented on a change in pull request #8358:
URL: https://github.com/apache/ignite/pull/8358#discussion_r520326008
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java
##
@@ -53,7 +53,7 @@ public GroupPartitionId(final int grpId, final int partId) {
* @return flag to be used for partition
*/
public static byte getFlagByPartId(final int partId) {
-return partId == PageIdAllocator.INDEX_PARTITION ? PageMemory.FLAG_IDX
: PageMemory.FLAG_DATA;
+return partId == PageIdAllocator.INDEX_PARTITION ? PageStore.TYPE_IDX
: PageStore.TYPE_DATA;
Review comment:
Additional static method for page store type created
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Wuchte commented on a change in pull request #8358: IGNITE-12489 Fixed race between B+Tree cursor and remove operation.
Wuchte commented on a change in pull request #8358:
URL: https://github.com/apache/ignite/pull/8358#discussion_r520337853
##
File path:
modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageIdUtilsSelfTest.java
##
@@ -99,13 +99,13 @@ public void testOffsetExtraction() throws Exception {
@Test
public void testPageIdFromLink() throws Exception {
assertEquals(0x00FFL,
PageIdUtils.pageId(0x00FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0x10FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0x01FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0x11FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0x80FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0x88FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0x08FFL));
-assertEquals(0x00FFL,
PageIdUtils.pageId(0xL));
+assertEquals(0x10FFL,
PageIdUtils.pageId(0x10FFL));
+assertEquals(0x01FFL,
PageIdUtils.pageId(0x01FFL));
+assertEquals(0x11FFL,
PageIdUtils.pageId(0x11FFL));
+assertEquals(0x80FFL,
PageIdUtils.pageId(0x80FFL));
+assertEquals(0x88FFL,
PageIdUtils.pageId(0x88FFL));
+assertEquals(0x08FFL,
PageIdUtils.pageId(0x08FFL));
+assertEquals(0xL,
PageIdUtils.pageId(0xL));
Review comment:
test updated
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Vladsz83 commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
Vladsz83 commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r520473812 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: I don't think so because socket is an internal implementation of the SPI. User shouldn't know about it. As example, there is no linger option for the communication SPI. Its internals are NIO. Linger was introduced only to have a workaround for SSL deadlock issues. It seems there was no need and are no need to configure it. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8432: IGNITE-13683 Support MVCC in ValidateIndexesClosure
asfgit closed pull request #8432: URL: https://github.com/apache/ignite/pull/8432 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8437: IGNITE-13686 Data structures test with enabled compression become failed.
asfgit closed pull request #8437: URL: https://github.com/apache/ignite/pull/8437 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] akalash commented on a change in pull request #8435: IGNITE-13684 PageIoResolver added
akalash commented on a change in pull request #8435:
URL: https://github.com/apache/ignite/pull/8435#discussion_r520400185
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
##
@@ -1561,7 +1593,7 @@ void decrementSize(int cacheId) {
return grp.mvccEnabled() ? dataTree.isEmpty() :
storageSize.get() == 0;
}
catch (IgniteCheckedException e) {
-U.error(log, "Failed to perform operation.", e);
+
U.error(grp.shared().logger(IgniteCacheOffheapManagerImpl.class), "Failed to
perform operation.", e);
Review comment:
Why don't you use log which was set in the constructor?
##
File path:
modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/inlinecolumn/ObjectHashInlineIndexColumn.java
##
@@ -60,7 +63,8 @@ public ObjectHashInlineIndexColumn(Column col) {
/** {@inheritDoc} */
@Override protected Value get0(long pageAddr, int off) {
-return null;
+int hashCode = PageUtils.getInt(pageAddr, off + 1);
+return new ValueObjectHashCode(hashCode); //TODO Is this ok?
Review comment:
TODO should be removed
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
##
@@ -639,6 +640,7 @@ else if (throttlingPlc ==
ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
* @return Data region configuration.
*/
private DataRegionConfiguration getDataRegionConfiguration() {
+//TODO
Review comment:
TODO?
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
##
@@ -2479,6 +2479,7 @@ private long tryToFindSequentially(int cap,
PageStoreWriter saveDirtyPage) throw
DataRegionConfiguration dataRegionCfg =
getDataRegionConfiguration();
+//TODO dataRegionCfg is null
Review comment:
TODO?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] ptupitsyn opened a new pull request #8445: IGNITE-13608 .NET: Add Partitions and UpdateBatchSize to SqlFieldsQuery
ptupitsyn opened a new pull request #8445: URL: https://github.com/apache/ignite/pull/8445 Add `Partitions` and `UpdateBatchSize` to `SqlFieldsQuery` and `QueryOptions` (LINQ) for thick and thin APIs This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] isapego closed pull request #8404: IGNITE-12126 ODBC SQLNumResultCols() works after SQLPrepare()
isapego closed pull request #8404: URL: https://github.com/apache/ignite/pull/8404 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] isapego commented on pull request #8404: IGNITE-12126 ODBC SQLNumResultCols() works after SQLPrepare()
isapego commented on pull request #8404: URL: https://github.com/apache/ignite/pull/8404#issuecomment-724951210 Merged to master. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] dmagda commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
dmagda commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r520867019 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: The linger setting is a TCP parameter that changes how the protocol behaves. Even if the initial motivation for adding it to the discovery SPI was to address the SSL issue, the setting still needs to be explained generically because it helps you to control the TCP behavior for all sorts of needs. The workarounds are usually handled via system properties in Ignite. As long as you've already added a new API method to the public interface it needs to be explained generically. Otherwise, you can remove the linger setting from the API and create a system property for the workaround instead. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] dmagda commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
dmagda commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r521048510 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: Moreover, you already have this workaround mentioned on the SSL page below that is a proper place. To recap, we need a generic description of this soLinger parameter on this page and have the workaround added as a warning callout on the SSL page. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ptupitsyn merged pull request #8443: IGNITE-13357 .NET: Add IncludeExpired to ContinuousQuery and ContinuousQueryClient
ptupitsyn merged pull request #8443: URL: https://github.com/apache/ignite/pull/8443 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ibessonov commented on a change in pull request #8435: IGNITE-13684 PageIoResolver added
ibessonov commented on a change in pull request #8435:
URL: https://github.com/apache/ignite/pull/8435#discussion_r521166094
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
##
@@ -1561,7 +1593,7 @@ void decrementSize(int cacheId) {
return grp.mvccEnabled() ? dataTree.isEmpty() :
storageSize.get() == 0;
}
catch (IgniteCheckedException e) {
-U.error(log, "Failed to perform operation.", e);
+
U.error(grp.shared().logger(IgniteCacheOffheapManagerImpl.class), "Failed to
perform operation.", e);
Review comment:
I don't know, I'll replace it with the filed access.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] xtern opened a new pull request #8446: IGNITE-13659 Cache encryption key rotation (documentation)
xtern opened a new pull request #8446: URL: https://github.com/apache/ignite/pull/8446 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8442: IGNITE-13665 When system worker is blocked, output its stack trace.
asfgit closed pull request #8442: URL: https://github.com/apache/ignite/pull/8442 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Vladsz83 commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
Vladsz83 commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r521307187 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: > The linger setting is a TCP parameter that changes how the protocol behaves. linger only sets socket behavior on socket closing with rest of the data in TCP buffers. But this doesn’t affect behavior of Discovery because it has own message acknowledgement. The exception is the delay of Discovery. That is why I think duplicating of Java API documentation is useless for Ignite user. This parameter doesn’t change general logic and gives no benefits to user. > The workarounds are usually handled via system properties in Ignite. As long as you've already added a new API method to the public interface . I haven't. This API had been added in 2.8 as hotfix of raised SSL issue. But hadn't been documented. We just changed it's default value to disabled like before 2.8. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8433: IGNITE-13681 Implemented non markers checkpoint implementation
asfgit closed pull request #8433: URL: https://github.com/apache/ignite/pull/8433 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] dmagda commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
dmagda commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r521359420 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: The value of the parameter [is passed into the Socket via its Java API](https://github.com/apache/ignite/blob/master/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java#L1687), thus the description on this page needs to sound generic like in Java. Our [JavaDoc follows this guideline](https://ignite.apache.org/releases/latest/javadoc/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.html#setSoLinger-int-) and doesn't mention any workarounds for specific use cases. We can put everything together this way mentioning the SSL issue as a reason why this parameter set to 0 for Ignite discovery sockets: _Specifies a linger-on-close timeout. This option disables/enables immediate return from a close() of a TCP Socket. The option defaults to `0` for Ignite TcpDiscovery sockets to avoid [potential deadlocks with SSL connections](https://bugs.openjdk.java.net/browse/JDK-8219658)._ Please take this all into consideration and send another pull-request. We've already beat this to death. The initial pull-request was merged skipping a review of technical documentation committers and maintainers. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] anton-vinogradov opened a new pull request #8447: IGNITE-13694 Check difference between SIGTERM, SIGKILL and Disconnect…
anton-vinogradov opened a new pull request #8447: URL: https://github.com/apache/ignite/pull/8447 … at Cellular switch Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Vladsz83 opened a new pull request #8448: IGNITE-13695 : Move javadoc of affection of several addresses on failure detection.
Vladsz83 opened a new pull request #8448: URL: https://github.com/apache/ignite/pull/8448 Better place for note of affection several node addresses on failure detection timeout. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #7890: IGNITE-13107: Fixed memory leak in ODBC
asfgit closed pull request #7890: URL: https://github.com/apache/ignite/pull/7890 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ptupitsyn merged pull request #8445: IGNITE-13608 .NET: Add Partitions and UpdateBatchSize to SqlFieldsQuery
ptupitsyn merged pull request #8445: URL: https://github.com/apache/ignite/pull/8445 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ibessonov closed pull request #8387: IGNITE-13190 for benchmark
ibessonov closed pull request #8387: URL: https://github.com/apache/ignite/pull/8387 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ibessonov opened a new pull request #8449: IGNITE-13697 Defragmentation control.sh API first commands.
ibessonov opened a new pull request #8449: URL: https://github.com/apache/ignite/pull/8449 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ivandasch opened a new pull request #8450: Fix regexp in baseline output.
ivandasch opened a new pull request #8450: URL: https://github.com/apache/ignite/pull/8450 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] anton-vinogradov merged pull request #8450: Fix regexp in baseline output.
anton-vinogradov merged pull request #8450: URL: https://github.com/apache/ignite/pull/8450 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8435: IGNITE-13684 PageIoResolver added
asfgit closed pull request #8435: URL: https://github.com/apache/ignite/pull/8435 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] agura opened a new pull request #8451: IGNITE-13701 Fixed content, order and amount of columns in INDEXES, TABLES and SCHEMAS views.
agura opened a new pull request #8451: URL: https://github.com/apache/ignite/pull/8451 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Vladsz83 commented on a change in pull request #8447: IGNITE-13694 Check difference between SIGTERM, SIGKILL and Disconnect…
Vladsz83 commented on a change in pull request #8447:
URL: https://github.com/apache/ignite/pull/8447#discussion_r522196062
##
File path: modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
##
@@ -200,16 +200,19 @@ def __exec_on_node(node, task, start_waiter=None,
delay_ms=0, time_holder=None):
task(node)
-def disconnect(self, nodes=None):
+def drop_network(self, nodes=None):
"""
Disconnects node from cluster.
"""
-self.logger.info("Disconnecting the node.")
+self.logger.info("Disconnecting the node(s)...")
if nodes is None:
assert self.num_nodes == 1
nodes = self.nodes
+for node in nodes:
+self.logger.debug("Disconnecting " + node.account.hostname + ".")
Review comment:
I suppose we would need this at info level. Next code provides on debug
"Activating netfilter on...". This also says which node ig going to get
stopped. But with extented netfilter info.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Vladsz83 commented on a change in pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
Vladsz83 commented on a change in pull request #8430: URL: https://github.com/apache/ignite/pull/8430#discussion_r521307187 ## File path: docs/_docs/clustering/network-configuration.adoc ## @@ -56,6 +56,7 @@ You can find the complete list of properties in the javadoc:org.apache.ignite.sp | `localPort` | The port that the node binds to. If set to a non-default value, other cluster nodes must know this port to be able to discover the node. | `47500` | `localPortRange`| If the `localPort` is busy, the node attempts to bind to the next port (incremented by 1) and continues this process until it finds a free port. The `localPortRange` property defines the number of ports the node will try (starting from `localPort`). | `100` +| `soLinger`| Setting linger-on-close can help with socket deadlocks of SSL issues like JDK-8219658. But costs longer detection of node failure. | `0` Review comment: > The linger setting is a TCP parameter that changes how the protocol behaves. linger only sets socket behavior on socket closing with rest of the data in TCP buffers. But this doesn’t affect behavior of Discovery because it has own message acknowledgement. The exception is the delay of Discovery. That is why I think duplicating of Java API documentation is useless for Ignite user. This parameter doesn’t change general logic and gives no benefits to user. > The workarounds are usually handled via system properties in Ignite. As long as you've already added a new API method to the public interface . I haven't. This API had been added in 2.8 as hotfix of raised SSL issue. But hadn't been documented. We just changed its default value to disabled like before 2.8. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Mmuzaf commented on a change in pull request #7984: IGNITE-13190
Mmuzaf commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r57175
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableBiFunction.java
##
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Represents a bi function that accepts one argument and produces a result.
Unlike {@link java.util.function.BiFunction}
+ * it is able to throw {@link IgniteCheckedException}.
+ *
+ * @param The type of the first input to the function.
+ * @param The type of the second input to the function.
+ * @param The type of the result of the function.
+ */
+public interface IgniteThrowableBiFunction extends Serializable {
Review comment:
Is this you are looking for `ThrowableBiFunction`?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Vladsz83 opened a new pull request #8452: IGNITE-13702 : Fix description of soLinger for DiscoveryTcpSpi.
Vladsz83 opened a new pull request #8452: URL: https://github.com/apache/ignite/pull/8452 Improoves description of soLinger property of DiscoverySpi. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Vladsz83 commented on pull request #8430: IGNITE-13662 : Describe soLinger setting in TCP Discovery and SSL issues.
Vladsz83 commented on pull request #8430: URL: https://github.com/apache/ignite/pull/8430#issuecomment-726226422 @dmagda , new doc improvement PR: https://github.com/apache/ignite/pull/8452 . Please review. Didn't only manage to make a caution within the table. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8403: IGNITE-13637: Implemented column-wise binding for result rows
asfgit closed pull request #8403: URL: https://github.com/apache/ignite/pull/8403 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite-website] mstekl opened a new pull request #40: udpated events until nov12, 2020
mstekl opened a new pull request #40: URL: https://github.com/apache/ignite-website/pull/40 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] isapego opened a new pull request #8453: IGNITE-13636: Fix ODBC Date metadata
isapego opened a new pull request #8453: URL: https://github.com/apache/ignite/pull/8453 Currently, Date collumns reported as SQL_BINARY by ODBC because we only recognize `java.util.Date` as SQL_DATE, not `java.sql.Date`. Added check, that if the column is of type `java.sql.Date` then report it as SQL_DATE also. Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite-website] dmagda merged pull request #40: udpated events until nov12, 2020
dmagda merged pull request #40: URL: https://github.com/apache/ignite-website/pull/40 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] anton-vinogradov merged pull request #8447: IGNITE-13694 Check difference between SIGTERM, SIGKILL and Disconnect…
anton-vinogradov merged pull request #8447: URL: https://github.com/apache/ignite/pull/8447 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8453: IGNITE-13636: Fix ODBC Date metadata
asfgit closed pull request #8453: URL: https://github.com/apache/ignite/pull/8453 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] anton-vinogradov opened a new pull request #8454: IGNITE-13703 Check difference between TcpDiscovery and ZookeeperDiscovery at Cellular switch
anton-vinogradov opened a new pull request #8454: URL: https://github.com/apache/ignite/pull/8454 … at Cellular switch Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] anton-vinogradov commented on pull request #8426: IGNITE-13666 : Fix detection of failed node number in the discovery ducktape test.
anton-vinogradov commented on pull request #8426: URL: https://github.com/apache/ignite/pull/8426#issuecomment-726687512 LGTM, Waiting for Jenkins recheck. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] asfgit closed pull request #8417: IGNITE-13655 Implement readiness probe REST endpoint
asfgit closed pull request #8417: URL: https://github.com/apache/ignite/pull/8417 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Vladsz83 opened a new pull request #8455: IGNITE-13704 : Try failuredetectionTimeout==500 in ducktape integration test.
Vladsz83 opened a new pull request #8455: URL: https://github.com/apache/ignite/pull/8455 Failure detection timeout is now 500ms. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] korlov42 opened a new pull request #8456: IGNITE-13544 Calcite integration. Merge joins
korlov42 opened a new pull request #8456: URL: https://github.com/apache/ignite/pull/8456 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] anton-vinogradov merged pull request #8426: IGNITE-13666 : Fix detection of failed node number in the discovery ducktape test.
anton-vinogradov merged pull request #8426: URL: https://github.com/apache/ignite/pull/8426 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Vladsz83 opened a new pull request #8457: IGNITE-13705 : Fix middle node failed when failed next node and previous.
Vladsz83 opened a new pull request #8457: URL: https://github.com/apache/ignite/pull/8457 The discovery ducktape test has detected failure of third node in the middle of 2 simulateously failed nodes. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ivandasch opened a new pull request #8458: IGNITE-13699 Register ZookeeperDiscovery metrics to new metrics framework, fix concurrency bugs.
ivandasch opened a new pull request #8458: URL: https://github.com/apache/ignite/pull/8458 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] gurustron opened a new pull request #8459: IGNITE-13336 - method translation error
gurustron opened a new pull request #8459: URL: https://github.com/apache/ignite/pull/8459 Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] dmagda merged pull request #8452: IGNITE-13702 : Fix description of soLinger for DiscoveryTcpSpi.
dmagda merged pull request #8452: URL: https://github.com/apache/ignite/pull/8452 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] samaitra opened a new pull request #8460: IGNITE-13539 Remove references for migrated Ignite Extensions modules…
samaitra opened a new pull request #8460: URL: https://github.com/apache/ignite/pull/8460 … from the assembly and osgi-karaf Thank you for submitting the pull request to the Apache Ignite. In order to streamline the review of the contribution we ask you to ensure the following steps have been taken: ### The Contribution Checklist - [ ] There is a single JIRA ticket related to the pull request. - [ ] The web-link to the pull request is attached to the JIRA ticket. - [ ] The JIRA ticket has the _Patch Available_ state. - [ ] The pull request body describes changes that have been made. The description explains _WHAT_ and _WHY_ was made instead of _HOW_. - [ ] The pull request title is treated as the final commit message. The following pattern must be used: `IGNITE- Change summary` where `` - number of JIRA issue. - [ ] A reviewer has been mentioned through the JIRA comments (see [the Maintainers list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers)) - [ ] The pull request has been checked by the Teamcity Bot and the `green visa` attached to the JIRA ticket (see [TC.Bot: Check PR](https://mtcga.gridgain.com/prs.html)) ### Notes - [How to Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute) - [Coding abbreviation rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules) - [Coding Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines) - [Apache Ignite Teamcity Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot) If you need any help, please email d...@ignite.apache.org or ask anу advice on http://asf.slack.com _#ignite_ channel. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r523996817
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableBiFunction.java
##
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Represents a bi function that accepts one argument and produces a result.
Unlike {@link java.util.function.BiFunction}
+ * it is able to throw {@link IgniteCheckedException}.
+ *
+ * @param The type of the first input to the function.
+ * @param The type of the second input to the function.
+ * @param The type of the result of the function.
+ */
+public interface IgniteThrowableBiFunction extends Serializable {
Review comment:
Exactly, seems like you found a dead code.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524009334 ## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java ## @@ -0,0 +1,887 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheType; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointState; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; +import org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; +import org.apache.ignite.internal.processors.cache.tree.CacheDataTree; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; +import org.apache.ignite.internal.processors.cache.tree.PendingRow; +import org.apache.ignite.internal.processors.query.GridQueryIndexing; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.collection.IntHashMap; +import org.apache.ignite.internal.util.collection.IntMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524014723 ## File path: modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java ## @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.defragmentation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.GridQueryIndexingDefragmentation; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.LinkMap; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.TimeTracker; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast; +import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataRow; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.database.H2Tree; +import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; +import org.apache.ignite.internal.processors.query.h2.database.InlineIndexColumn; +import org.apache.ignite.internal.processors.query.h2.database.inlinecolumn.AbstractInlineIndexColumn; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2ExtrasInnerIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2ExtrasLeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2InnerIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2LeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.H2RowLinkIO; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; +import org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow; +import org.apache.ignite.internal.processors.query.h2.opt.H2Row; +import org.apache.ignite.internal.util.collection.IntMap; +import org.h2.index.Index; +import org.h2.value.Value; + +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.CP_LOCK; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.INIT_TREE; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.INSERT_ROW; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.ITERATE; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentat
[GitHub] [ignite] Mmuzaf commented on a change in pull request #7984: IGNITE-13190
Mmuzaf commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524039648
##
File path:
modules/core/src/main/java/org/apache/ignite/configuration/DefragmentationConfiguration.java
##
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+public final class DefragmentationConfiguration implements Serializable {
Review comment:
Do we need a dedicated `DefragmentationConfiguration`? Can we store all
these properties in `DataStorageConfiguration`?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r524134745
##
File path: modules/ducktests/tests/ignitetest/tests/snapshot_test.py
##
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module contains discovery tests.
+"""
+from ducktape.mark.resource import cluster
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.ignite_app import IgniteApplicationService
+from ignitetest.services.utils.control_utility import ControlUtility
+from ignitetest.services.utils.ignite_configuration import
IgniteConfiguration, DataStorageConfiguration
+from ignitetest.services.utils.ignite_configuration.data_storage import
DataRegionConfiguration
+from ignitetest.services.utils.ignite_configuration.discovery import
from_ignite_cluster
+from ignitetest.utils import ignite_versions
+from ignitetest.utils.ignite_test import IgniteTest
+from ignitetest.utils.version import DEV_BRANCH, IgniteVersion
+
+
+# pylint: disable=W0223
+class SnapshotTest(IgniteTest):
+"""
+Test Snapshot.
+"""
+NUM_NODES = 4
+
+SNAPSHOT_NAME = "test_snap"
+
+@cluster(num_nodes=NUM_NODES)
+@ignite_versions(str(DEV_BRANCH))
+def snapshot_test(self, ignite_version):
+"""
+Test Snapshot.
+1. Start of ignite cluster.
+2. Activate cluster.
+3. Load.
+4. Idle verify dump_1.
+5. Snapshot.
+6. Load.
+7. Idle verify dump_2.
+8. Сhecking that dump_1 and dump_2 are different.
+9. Stop ignite and replace db from snapshot.
+10. Run ignite cluster.
+11. Idle verify dump_3.
+12. Checking the equality of dump_1 and dump_3.
+"""
+data_storage =
DataStorageConfiguration(default=DataRegionConfiguration(persistent=True))
+
+ignite_config = IgniteConfiguration(
+version=IgniteVersion(ignite_version),
+data_storage=data_storage
+)
+
+service = IgniteService(self.test_context, ignite_config,
num_nodes=self.NUM_NODES - 1)
+service.start()
+
+control_utility = ControlUtility(service, self.test_context)
+control_utility.activate()
+
+client_config = IgniteConfiguration(
+client_mode=True,
+version=IgniteVersion(ignite_version),
+discovery_spi=from_ignite_cluster(service),
+)
+
+streamer = IgniteApplicationService(
+self.test_context,
+client_config,
+
java_class_name="org.apache.ignite.internal.ducktest.tests.UuidDataStreamerApplication",
+params={
+"cacheName": "test-cache",
+"iterSize": 512 * 1024
+}
+)
+
+load(streamer)
+
+node = service.nodes[0]
+
+control_utility.validate_indexes(check_assert=True)
+control_utility.idle_verify(check_assert=True)
+dump_1 = control_utility.idle_verify_dump(node, return_path=True)
+
+self.logger.info(f'Path to dump_1 on
{node.account.externally_routable_ip}={dump_1}')
+
+control_utility.snapshot_create(self.SNAPSHOT_NAME)
+
+load(streamer)
+
+dump_2 = control_utility.idle_verify_dump(node, return_path=True)
+
+self.logger.info(f'Path to dump_2 on
{node.account.externally_routable_ip}={dump_2}')
+
+diff = node.account.ssh_output(f'diff {dump_1} {dump_2}',
allow_fail=True)
+assert len(diff) != 0
+
+service.stop()
+
+service.rename_db(new_db_name='old_db')
+service.restore_from_snapshot(self.SNAPSHOT_NAME)
+
+service.restart()
+
+control_utility.activate()
+
+control_utility.validate_indexes(check_assert=True)
+control_utility.idle_verify(check_assert=True)
+dump_3 = control_utility.idle_verify_dump(node, return_path=True)
+
+self.logger.info(f'Path to dump_3 on
{node.account.externally_routable_ip}={dump_3}')
+
+diff = node.account.ssh_output(f'diff {dump_1} {dump_3}',
allow_fail=True)
+assert len(diff) == 0, diff
+
+
+def load(service_load: IgniteApplicationService, duration: int = 300):
+"""
+Load.
[GitHub] [ignite-extensions] ololo3000 commented on pull request #28: IGNITE-13634 Changes scope of Ignite and integration target dependencies to provided.
ololo3000 commented on pull request #28: URL: https://github.com/apache/ignite-extensions/pull/28#issuecomment-727912699 @samaitra I would appreciate it if you would take another look. I expanded the PR as discussed [1]. [1] - http://apache-ignite-developers.2346864.n4.nabble.com/DISCUSS-Release-Ignite-streamer-extensions-td49858.html This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r524156096
##
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java
##
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests;
+
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Loading random uuids to cache.
+ */
+public class UuidDataStreamerApplication extends IgniteAwareApplication {
+/** {@inheritDoc} */
+@Override public void run(JsonNode jNode) throws InterruptedException {
+String cacheName = jNode.get("cacheName").asText();
+
+int dataSize = Optional.ofNullable(jNode.get("dataSize"))
+.map(JsonNode::asInt)
+.orElse(1024);
+
+long iterSize = Optional.ofNullable(jNode.get("iterSize"))
+.map(JsonNode::asLong)
+.orElse(1024L);
+
+assert dataSize > 0;
+assert iterSize > 0;
+
+CacheConfiguration cacheCfg = new
CacheConfiguration<>(cacheName);
+cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+cacheCfg.setBackups(2);
+cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+cacheCfg.setIndexedTypes(UUID.class, byte[].class);
+
+ignite.getOrCreateCache(cacheCfg);
+
+long start = System.currentTimeMillis();
+
+markInitialized();
+
+workParallel(ignite, cacheName, iterSize, dataSize);
+
+recordResult("DURATION", System.currentTimeMillis() - start);
+
+markFinished();
+}
+
+/** */
+private void workParallel(Ignite ignite, String cacheName, long iterSize,
int dataSize)
+throws InterruptedException {
+int threads = Runtime.getRuntime().availableProcessors() / 2;
+
+long iterThread = iterSize / threads;
Review comment:
number of iteration per thread
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r524156395
##
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java
##
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests;
+
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Loading random uuids to cache.
+ */
+public class UuidDataStreamerApplication extends IgniteAwareApplication {
+/** {@inheritDoc} */
+@Override public void run(JsonNode jNode) throws InterruptedException {
+String cacheName = jNode.get("cacheName").asText();
+
+int dataSize = Optional.ofNullable(jNode.get("dataSize"))
+.map(JsonNode::asInt)
+.orElse(1024);
+
+long iterSize = Optional.ofNullable(jNode.get("iterSize"))
+.map(JsonNode::asLong)
+.orElse(1024L);
+
+assert dataSize > 0;
+assert iterSize > 0;
+
+CacheConfiguration cacheCfg = new
CacheConfiguration<>(cacheName);
+cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+cacheCfg.setBackups(2);
+cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
Review comment:
fixed
##
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java
##
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests;
+
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Loading random uuids to cache.
+ */
+public class UuidDataStreamerApplication extends IgniteAwareApplication {
+/** {@inheritDoc} */
+@Override public void run(JsonNode jNode) throws InterruptedException {
+String cacheName = jNode.get("cacheName").asText();
+
+int dataSize = Optional.ofNullable(jNode.get("dataSize"))
+.map(JsonNode::asInt)
+.orElse(1024);
+
+long iterSize = Optional.ofNullable(jNode.get("iterSize"))
+.map(JsonNode::asLong)
+.orElse(1024L);
+
+assert dataSize > 0;
+assert iterSize > 0;
+
+CacheConfiguration cacheCfg = new
Ca
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524281992 ## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java ## @@ -0,0 +1,887 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheType; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointState; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; +import org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; +import org.apache.ignite.internal.processors.cache.tree.CacheDataTree; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; +import org.apache.ignite.internal.processors.cache.tree.PendingRow; +import org.apache.ignite.internal.processors.query.GridQueryIndexing; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.collection.IntHashMap; +import org.apache.ignite.internal.util.collection.IntMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524285281
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/DefragmentationFileUtils.java
##
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.ignite.internal.processors.cache.persistence.defragmentation;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import
org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static java.nio.file.StandardOpenOption.CREATE_NEW;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.FILE_SUFFIX;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_PREFIX;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_SUFFIX;
+
+/**
+ * Everything related to file management during defragmentation process.
+ */
+public class DefragmentationFileUtils {
+/** Prefix for link mapping files. */
+private static final String DFRG_LINK_MAPPING_FILE_PREFIX =
PART_FILE_PREFIX + "map-";
+
+/** Link mapping file template. */
+private static final String DFRG_LINK_MAPPING_FILE_TEMPLATE =
DFRG_LINK_MAPPING_FILE_PREFIX + "%d" + FILE_SUFFIX;
+
+/** Defragmentation complation marker file name. */
+private static final String DFRG_COMPLETION_MARKER_FILE_NAME =
"dfrg-completion-marker";
+
+/** Name of defragmentated index partition file. */
+private static final String DFRG_INDEX_FILE_NAME = INDEX_FILE_PREFIX +
"-dfrg" + FILE_SUFFIX;
+
+/** Name of defragmentated index partition temporary file. */
+private static final String DFRG_INDEX_TMP_FILE_NAME =
DFRG_INDEX_FILE_NAME + TMP_SUFFIX;
+
+/** Prefix for defragmented partition files. */
+private static final String DFRG_PARTITION_FILE_PREFIX = PART_FILE_PREFIX
+ "dfrg-";
+
+/** Defragmented partition file template. */
+private static final String DFRG_PARTITION_FILE_TEMPLATE =
DFRG_PARTITION_FILE_PREFIX + "%d" + FILE_SUFFIX;
+
+/** Defragmented partition temp file template. */
+private static final String DFRG_PARTITION_TMP_FILE_TEMPLATE =
DFRG_PARTITION_FILE_TEMPLATE + TMP_SUFFIX;
+
+/**
+ * Performs cleanup of work dir before initializing file page stores.
+ * Will finish batch renaming if defragmentation was completed or delete
garbage if it wasn't.
+ *
+ * @param workDir Cache group working directory.
+ * @param log Logger to write messages.
+ * @throws IgniteCheckedException If {@link IOException} occurred.
+ */
+public static void beforeInitPageStores(File workDir, IgniteLogger log)
throws IgniteCheckedException {
+batchRenameDefragmentedCacheGroupPartitions(workDir, log);
+
+U.delete(defragmentationCompletionMarkerFile(workDir));
+
+for (File file : workDir.listFiles()) {
+String fileName = file.getName();
+
+if (
+fileName.startsWith(DFRG_PARTITION_FILE_PREFIX)
+|| fileName.startsWith(DFRG_INDEX_FILE_NAME)
+|| fileName.startsWith(DFRG_LINK_MAPPING_FILE_PREFIX)
+)
+U.delete(file);
+}
+}
+
+/**
+
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524294661 ## File path: modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java ## @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.defragmentation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.GridQueryIndexingDefragmentation; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.LinkMap; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.TimeTracker; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast; +import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataRow; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.database.H2Tree; +import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; +import org.apache.ignite.internal.processors.query.h2.database.InlineIndexColumn; +import org.apache.ignite.internal.processors.query.h2.database.inlinecolumn.AbstractInlineIndexColumn; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2ExtrasInnerIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2ExtrasLeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2InnerIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2LeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.H2RowLinkIO; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; +import org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow; +import org.apache.ignite.internal.processors.query.h2.opt.H2Row; +import org.apache.ignite.internal.util.collection.IntMap; +import org.h2.index.Index; +import org.h2.value.Value; + +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.CP_LOCK; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.INIT_TREE; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.INSERT_ROW; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.ITERATE; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmenta
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524301405 ## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java ## @@ -0,0 +1,887 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheType; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointState; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; +import org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; +import org.apache.ignite.internal.processors.cache.tree.CacheDataTree; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; +import org.apache.ignite.internal.processors.cache.tree.PendingRow; +import org.apache.ignite.internal.processors.query.GridQueryIndexing; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.collection.IntHashMap; +import org.apache.ignite.internal.util.collection.IntMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524310015
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
##
@@ -1678,6 +1678,51 @@ public static String toString(String str,
}
}
+/**
+ * Produces uniformed output of string with context properties
+ *
+ * @param str Output prefix or {@code null} if empty.
+ * @param triplets Triplets {@code {name, value, sencitivity}}.
+ * @return String presentation.
+ */
+public static String toString(String str, Object... triplets) {
+assert triplets.length % 3 == 0;
Review comment:
Looks like this should be an exception
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524315198
##
File path:
modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsIndexingDefragmentationTest.java
##
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.function.Function;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils;
+import
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.verify.ValidateIndexesClosure;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.junit.Test;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static
org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Defragmentation tests with enabled ignite-indexing.
+ */
+public class IgnitePdsIndexingDefragmentationTest extends
IgnitePdsDefragmentationTest {
+/** Use MVCC in tests. */
+private static final String USE_MVCC = "USE_MVCC";
+
+/** {@inheritDoc} */
+@Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+cfg.setConsistentId(igniteInstanceName);
+
+DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+dsCfg.setWalSegmentSize(4 * 1024 * 1024);
+
+dsCfg.setDefaultDataRegionConfiguration(
+new DataRegionConfiguration()
+.setInitialSize(100L * 1024 * 1024)
+.setMaxSize(1024L * 1024 * 1024)
+.setPersistenceEnabled(true)
+);
+
+cfg.setDataStorageConfiguration(dsCfg);
+
+CacheConfiguration cache1Cfg = new
CacheConfiguration<>(DEFAULT_CACHE_NAME)
+.setAtomicityMode(TRANSACTIONAL)
+.setGroupName(GRP_NAME)
+.setIndexedTypes(
+ObjKey.class, byte[].class,
+Integer.class, byte[].class
+)
+.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+CacheConfiguration cache2Cfg = new
CacheConfiguration<>(CACHE_2_NAME)
+.setAtomicityMode(TRANSACTIONAL)
+.setGroupName(GRP_NAME)
+.setIndexedTypes(
+ObjKey.class, byte[].class,
+Integer.class, byte[].class
Review comment:
Need to add more tests for actual complex indexes: multi-column, various
inline sizes.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524315640
##
File path:
modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsIndexingDefragmentationTest.java
##
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.function.Function;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils;
+import
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.verify.ValidateIndexesClosure;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.junit.Test;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static
org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Defragmentation tests with enabled ignite-indexing.
+ */
+public class IgnitePdsIndexingDefragmentationTest extends
IgnitePdsDefragmentationTest {
+/** Use MVCC in tests. */
+private static final String USE_MVCC = "USE_MVCC";
+
+/** {@inheritDoc} */
+@Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+cfg.setConsistentId(igniteInstanceName);
+
+DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+dsCfg.setWalSegmentSize(4 * 1024 * 1024);
+
+dsCfg.setDefaultDataRegionConfiguration(
+new DataRegionConfiguration()
+.setInitialSize(100L * 1024 * 1024)
+.setMaxSize(1024L * 1024 * 1024)
+.setPersistenceEnabled(true)
+);
+
+cfg.setDataStorageConfiguration(dsCfg);
+
+CacheConfiguration cache1Cfg = new
CacheConfiguration<>(DEFAULT_CACHE_NAME)
+.setAtomicityMode(TRANSACTIONAL)
+.setGroupName(GRP_NAME)
+.setIndexedTypes(
+ObjKey.class, byte[].class,
+Integer.class, byte[].class
+)
+.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+CacheConfiguration cache2Cfg = new
CacheConfiguration<>(CACHE_2_NAME)
+.setAtomicityMode(TRANSACTIONAL)
+.setGroupName(GRP_NAME)
+.setIndexedTypes(
+ObjKey.class, byte[].class,
+Integer.class, byte[].class
+)
+.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+if (Boolean.TRUE.toString().equals(System.getProperty(USE_MVCC))) {
+cache1Cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT);
+cache2Cfg.setAtomicityMode(TRANSACTIONAL_SNAPSHOT);
+} else
+cache2Cfg.setExpiryPolicyFactory(new PolicyFactory());
+
+cfg.setCacheConfiguration(cache1Cfg, cache2Cfg);
+
+return cfg;
+}
+
+/** {@inheritDoc} */
+@Override protected void afterTest() throws Exception
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524319380 ## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java ## @@ -0,0 +1,887 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheType; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointState; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; +import org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; +import org.apache.ignite.internal.processors.cache.tree.CacheDataTree; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; +import org.apache.ignite.internal.processors.cache.tree.PendingRow; +import org.apache.ignite.internal.processors.query.GridQueryIndexing; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.collection.IntHashMap; +import org.apache.ignite.internal.util.collection.IntMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524320192
##
File path:
modules/core/src/main/java/org/apache/ignite/configuration/DefragmentationConfiguration.java
##
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+public final class DefragmentationConfiguration implements Serializable {
Review comment:
Yes, we can, it's a subject for discussion I guess.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524319939 ## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java ## @@ -0,0 +1,887 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheType; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointState; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; +import org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; +import org.apache.ignite.internal.processors.cache.tree.CacheDataTree; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; +import org.apache.ignite.internal.processors.cache.tree.PendingRow; +import org.apache.ignite.internal.processors.query.GridQueryIndexing; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.collection.IntHashMap; +import org.apache.ignite.internal.util.collection.IntMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524322515 ## File path: modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java ## @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.defragmentation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.GridQueryIndexingDefragmentation; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.LinkMap; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.TimeTracker; +import org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver; +import org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast; +import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataRow; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.database.H2Tree; +import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; +import org.apache.ignite.internal.processors.query.h2.database.InlineIndexColumn; +import org.apache.ignite.internal.processors.query.h2.database.inlinecolumn.AbstractInlineIndexColumn; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2ExtrasInnerIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2ExtrasLeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2InnerIO; +import org.apache.ignite.internal.processors.query.h2.database.io.AbstractH2LeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.H2RowLinkIO; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; +import org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow; +import org.apache.ignite.internal.processors.query.h2.opt.H2Row; +import org.apache.ignite.internal.util.collection.IntMap; +import org.h2.index.Index; +import org.h2.value.Value; + +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.CP_LOCK; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.INIT_TREE; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.INSERT_ROW; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentation.IndexStages.ITERATE; +import static org.apache.ignite.internal.processors.query.h2.defragmentation.IndexingDefragmentat
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524322120
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/DefragmentationFileUtils.java
##
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.ignite.internal.processors.cache.persistence.defragmentation;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import
org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static java.nio.file.StandardOpenOption.CREATE_NEW;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.FILE_SUFFIX;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_PREFIX;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
+import static
org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_SUFFIX;
+
+/**
+ * Everything related to file management during defragmentation process.
+ */
+public class DefragmentationFileUtils {
+/** Prefix for link mapping files. */
+private static final String DFRG_LINK_MAPPING_FILE_PREFIX =
PART_FILE_PREFIX + "map-";
+
+/** Link mapping file template. */
+private static final String DFRG_LINK_MAPPING_FILE_TEMPLATE =
DFRG_LINK_MAPPING_FILE_PREFIX + "%d" + FILE_SUFFIX;
+
+/** Defragmentation complation marker file name. */
+private static final String DFRG_COMPLETION_MARKER_FILE_NAME =
"dfrg-completion-marker";
+
+/** Name of defragmentated index partition file. */
+private static final String DFRG_INDEX_FILE_NAME = INDEX_FILE_PREFIX +
"-dfrg" + FILE_SUFFIX;
+
+/** Name of defragmentated index partition temporary file. */
+private static final String DFRG_INDEX_TMP_FILE_NAME =
DFRG_INDEX_FILE_NAME + TMP_SUFFIX;
+
+/** Prefix for defragmented partition files. */
+private static final String DFRG_PARTITION_FILE_PREFIX = PART_FILE_PREFIX
+ "dfrg-";
+
+/** Defragmented partition file template. */
+private static final String DFRG_PARTITION_FILE_TEMPLATE =
DFRG_PARTITION_FILE_PREFIX + "%d" + FILE_SUFFIX;
+
+/** Defragmented partition temp file template. */
+private static final String DFRG_PARTITION_TMP_FILE_TEMPLATE =
DFRG_PARTITION_FILE_TEMPLATE + TMP_SUFFIX;
+
+/**
+ * Performs cleanup of work dir before initializing file page stores.
+ * Will finish batch renaming if defragmentation was completed or delete
garbage if it wasn't.
+ *
+ * @param workDir Cache group working directory.
+ * @param log Logger to write messages.
+ * @throws IgniteCheckedException If {@link IOException} occurred.
+ */
+public static void beforeInitPageStores(File workDir, IgniteLogger log)
throws IgniteCheckedException {
+batchRenameDefragmentedCacheGroupPartitions(workDir, log);
+
+U.delete(defragmentationCompletionMarkerFile(workDir));
+
+for (File file : workDir.listFiles()) {
+String fileName = file.getName();
+
+if (
+fileName.startsWith(DFRG_PARTITION_FILE_PREFIX)
+|| fileName.startsWith(DFRG_INDEX_FILE_NAME)
+|| fileName.startsWith(DFRG_LINK_MAPPING_FILE_PREFIX)
+)
+U.delete(file);
+}
+}
+
+/**
+
[GitHub] [ignite] Mmuzaf commented on a change in pull request #7984: IGNITE-13190
Mmuzaf commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524324305
##
File path:
modules/core/src/main/java/org/apache/ignite/configuration/DefragmentationConfiguration.java
##
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+public final class DefragmentationConfiguration implements Serializable {
Review comment:
I suggest minimizing changes. If the wider defragmentation configuration
scope appears we can create a such a class configuration, but currently, I see
no reasons.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r524325758 ## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java ## @@ -0,0 +1,887 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.store.PageStore; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheType; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CheckpointState; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock; +import org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList; +import org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; +import org.apache.ignite.internal.processors.cache.tree.CacheDataTree; +import org.apache.ignite.internal.processors.cache.tree.DataRow; +import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree; +import org.apache.ignite.internal.processors.cache.tree.PendingRow; +import org.apache.ignite.internal.processors.query.GridQueryIndexing; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.collection.IntHashMap; +import org.apache.ignite.internal.util.collection.IntMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.
[GitHub] [ignite] ibessonov commented on a change in pull request #7984: IGNITE-13190
ibessonov commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524327899
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
##
@@ -1678,6 +1678,51 @@ public static String toString(String str,
}
}
+/**
+ * Produces uniformed output of string with context properties
+ *
+ * @param str Output prefix or {@code null} if empty.
+ * @param triplets Triplets {@code {name, value, sencitivity}}.
+ * @return String presentation.
+ */
+public static String toString(String str, Object... triplets) {
+assert triplets.length % 3 == 0;
Review comment:
Ok, I'll change it.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] agoncharuk commented on a change in pull request #7984: IGNITE-13190
agoncharuk commented on a change in pull request #7984:
URL: https://github.com/apache/ignite/pull/7984#discussion_r524330324
##
File path:
modules/core/src/main/java/org/apache/ignite/configuration/DefragmentationConfiguration.java
##
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+public final class DefragmentationConfiguration implements Serializable {
+/** Serial version uid. */
+private static final long serialVersionUID = 0L;
+
+/**
+ *
+ */
+public static final long DFLT_MAPPIN_REGION_SIZE = 256L * 1024 * 1024;
+
+/**
+ *
+ */
+private long regionSize =
DataStorageConfiguration.DFLT_DATA_REGION_MAX_SIZE;
+
+/**
+ *
+ */
+private long mappingRegionSize = DFLT_MAPPIN_REGION_SIZE;
Review comment:
Can we calculate defragmentation regions automatically? For example, the
mapping region size can be calculated as max(size of the cache) * C (we store a
link->link mapping, so the region size can be estimated pretty much
accurately).
For the defragmentation region size, we can calculate DS = Sum(all data
region sizes in the node configuration). Then, assuming defragmentation runs
sequentially per cache/cache group, we can split the DS proportionally between
the original cache data region, the defragmentation data region, and mapping
data region.
If this is something viable, I would move these configuration parameters to
system properties so that user does not have to configure anything at all.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8402: IGNITE-13510: Getting status of snapshot execution via command line and jmx.
Sega76 commented on a change in pull request #8402:
URL: https://github.com/apache/ignite/pull/8402#discussion_r524487500
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotStatusTask.java
##
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.snapshot;
+
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSnapshot;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * @see IgniteSnapshot#statusSnapshot()
+ */
+@GridInternal
+public class VisorSnapshotStatusTask extends VisorOneNodeTask {
+/** Serial version uid. */
+private static final long serialVersionUID = 0L;
+
+/** {@inheritDoc} */
+@Override protected VisorJob job(Void arg) {
+return new VisorSnapshotStatusJob(debug);
+}
+
+/** */
+private static class VisorSnapshotStatusJob extends VisorJob
{
+/** Serial version uid. */
+private static final long serialVersionUID = 0L;
+
+/**
+ * @param debug Flag indicating whether debug information should be
printed into node log.
+ */
+protected VisorSnapshotStatusJob(boolean debug) {
+super(null, debug);
+}
+
+/** {@inheritDoc} */
+@Override protected String run(Void arg) throws IgniteException {
+Set ids =
ignite.context().cache().context().snapshotMgr().statusSnapshot().get().stream()
+.filter(Objects::nonNull)
+.collect(Collectors.toSet());
+
+if (ids.isEmpty())
+return "No snapshot operations.";
+
+StringBuilder sb = new StringBuilder("чSnapshot operation in
progress on nodes with Consistent ID:");
Review comment:
fixed
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8402: IGNITE-13510: Getting status of snapshot execution via command line and jmx.
Sega76 commented on a change in pull request #8402:
URL: https://github.com/apache/ignite/pull/8402#discussion_r524487750
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
##
@@ -1456,6 +1470,29 @@ public CancelSnapshotCallable(String snpName) {
}
}
+/** Get the status of a cluster snapshot operation. */
+@GridInternal
+private static class StatusSnapshotCallable implements IgniteClosure {
+/** Serial version UID. */
+private static final long serialVersionUID = 0L;
+
+/** Auto-injected grid instance. */
+@IgniteInstanceResource
+private transient IgniteEx ignite;
+
+/** */
+public StatusSnapshotCallable() {
+}
+
+/** {@inheritDoc} */
+@Override public Object apply(Void unused) {
+if
(ignite.context().cache().context().snapshotMgr().isSnapshotCreating())
Review comment:
fixed
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java
##
@@ -47,4 +50,11 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) {
@Override public void cancelSnapshot(String snpName) {
mgr.cancelSnapshot(snpName).get();
}
+
+/** {@inheritDoc} */
+@Override public Collection statusSnapshot() {
+return mgr.statusSnapshot().get().stream()
Review comment:
fixed
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8402: IGNITE-13510: Getting status of snapshot execution via command line and jmx.
Sega76 commented on a change in pull request #8402:
URL: https://github.com/apache/ignite/pull/8402#discussion_r524488066
##
File path:
modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/snapshot/SnapshotCommand.java
##
@@ -51,13 +53,18 @@
/** {@inheritDoc} */
@Override public Object execute(GridClientConfiguration clientCfg, Logger
log) throws Exception {
try (GridClient client = Command.startClient(clientCfg)) {
-return executeTaskByNameOnNode(
-client,
-taskName,
-taskArgs,
-null,
-clientCfg
+Object res = executeTaskByNameOnNode(
+client,
+taskName,
+taskArgs,
+null,
+clientCfg
);
+
+if (taskName.equals(VisorSnapshotStatusTask.class.getName()))
Review comment:
fixed
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8402: IGNITE-13510: Getting status of snapshot execution via command line and jmx.
Sega76 commented on a change in pull request #8402:
URL: https://github.com/apache/ignite/pull/8402#discussion_r524488550
##
File path:
modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
##
@@ -2293,6 +2293,41 @@ public void testClusterSnapshotCreate() throws Exception
{
assertTrue("Snapshot must contains cache data [left=" + range + ']',
range.isEmpty());
}
+/** @throws Exception If failed. */
+@Test
+public void testClusterSnapshotStatus() throws Exception {
+int keysCnt = 100;
+String snpName = "snapshot_02052020";
+
+IgniteEx ig = startGrid(0);
+ig.cluster().state(ACTIVE);
+
+createCacheAndPreload(ig, keysCnt);
+
+CommandHandler h = new CommandHandler();
+
+assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "status"));
+
+assertTrue(h.getLastOperationResult().toString().contains("No snapshot
operations."));
+
+assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "create",
snpName));
+
+assertEquals(EXIT_CODE_OK, execute(h, "--snapshot", "status"));
+
+assertTrue(h.getLastOperationResult().toString()
+.contains("Snapshot operation in progress on nodes with
Consistent ID:"));
Review comment:
added assert
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8402: IGNITE-13510: Getting status of snapshot execution via command line and jmx.
Sega76 commented on a change in pull request #8402:
URL: https://github.com/apache/ignite/pull/8402#discussion_r524491310
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/visor/snapshot/VisorSnapshotStatusTask.java
##
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.snapshot;
+
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSnapshot;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * @see IgniteSnapshot#statusSnapshot()
+ */
+@GridInternal
+public class VisorSnapshotStatusTask extends VisorOneNodeTask {
+/** Serial version uid. */
+private static final long serialVersionUID = 0L;
+
+/** {@inheritDoc} */
+@Override protected VisorJob job(Void arg) {
+return new VisorSnapshotStatusJob(debug);
+}
+
+/** */
+private static class VisorSnapshotStatusJob extends VisorJob
{
+/** Serial version uid. */
+private static final long serialVersionUID = 0L;
+
+/**
+ * @param debug Flag indicating whether debug information should be
printed into node log.
+ */
+protected VisorSnapshotStatusJob(boolean debug) {
+super(null, debug);
+}
+
+/** {@inheritDoc} */
+@Override protected String run(Void arg) throws IgniteException {
+Set ids =
ignite.context().cache().context().snapshotMgr().statusSnapshot().get().stream()
Review comment:
fixed
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8402: IGNITE-13510: Getting status of snapshot execution via command line and jmx.
Sega76 commented on a change in pull request #8402:
URL: https://github.com/apache/ignite/pull/8402#discussion_r524493773
##
File path:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
##
@@ -1456,6 +1470,29 @@ public CancelSnapshotCallable(String snpName) {
}
}
+/** Get the status of a cluster snapshot operation. */
+@GridInternal
+private static class StatusSnapshotCallable implements IgniteClosure {
Review comment:
use `VisorMultiNodeTask`
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294: URL: https://github.com/apache/ignite/pull/8294#discussion_r524743704 ## File path: modules/ducktests/tests/ignitetest/tests/snapshot_test.py ## @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module contains discovery tests. +""" +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class SnapshotTest(IgniteTest): +""" +Test Snapshot. +""" +NUM_NODES = 5 + +SNAPSHOT_NAME = "test_snap" + +CACHE_NAME = "TEST_CACHE" + +@cluster(num_nodes=NUM_NODES) +@ignite_versions(str(DEV_BRANCH)) +def snapshot_test(self, ignite_version): +""" +Basic snapshot test. +""" +data_storage = DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)) + +ignite_config = IgniteConfiguration( +version=IgniteVersion(ignite_version), +data_storage=data_storage, +caches=[CacheConfiguration(name=self.CACHE_NAME, backups=2, indexed_types=['java.util.UUID', 'byte[]'])] Review comment: with backups: - we need to load less data for the desired size, - index validation is faster, - the test runs faster. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [ignite-extensions] dmagda commented on a change in pull request #30: IGNITE-12951 Update documents for migrated extensions
dmagda commented on a change in pull request #30:
URL: https://github.com/apache/ignite-extensions/pull/30#discussion_r524745789
##
File path: modules/flink-ext/README.txt
##
@@ -20,7 +20,7 @@ interested in):
...
org.apache.ignite
-ignite-flink
+ignite-flink-ext
${ignite.version}
Review comment:
@samaitra, sorry for a belated check. All looks good to me, please go
ahead and merge the changes to the master.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] Sega76 commented on a change in pull request #8294: IGNITE-13492: Basic snapshot test
Sega76 commented on a change in pull request #8294:
URL: https://github.com/apache/ignite/pull/8294#discussion_r524156512
##
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/UuidDataStreamerApplication.java
##
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests;
+
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Loading random uuids to cache.
+ */
+public class UuidDataStreamerApplication extends IgniteAwareApplication {
+/** {@inheritDoc} */
+@Override public void run(JsonNode jNode) throws InterruptedException {
+String cacheName = jNode.get("cacheName").asText();
+
+int dataSize = Optional.ofNullable(jNode.get("dataSize"))
+.map(JsonNode::asInt)
+.orElse(1024);
+
+long iterSize = Optional.ofNullable(jNode.get("iterSize"))
+.map(JsonNode::asLong)
+.orElse(1024L);
+
+assert dataSize > 0;
+assert iterSize > 0;
+
+CacheConfiguration cacheCfg = new
CacheConfiguration<>(cacheName);
+cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+cacheCfg.setBackups(2);
Review comment:
with backups:
- we need to load less data for the desired size,
- index validation is faster,
- the test runs faster
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [ignite] dmagda commented on pull request #8424: IGNITE-13663 : Represent in the documenttion affection of several node addresses on failure detection v2.
dmagda commented on pull request #8424: URL: https://github.com/apache/ignite/pull/8424#issuecomment-728422124 @Vladsz83, should we encourage the developer to initialize the TcpCommunicationSpi.localAddresses as well? So that the nodes use only reachable network interfaces for communication needs. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
