This is an automated email from the ASF dual-hosted git repository. nizhikov pushed a commit to branch IGNITE-19950-snapshot-merge in repository https://gitbox.apache.org/repos/asf/ignite.git
commit be062a79f20e5304191d8661be6350e6a811b9a7 Author: nizhikov <[email protected]> AuthorDate: Fri Jul 28 18:47:26 2023 +0300 IGNITE-19950 Dump creation process implemented --- .../snapshot/IgniteSnapshotManager.java | 46 +++++++---- .../snapshot/dump/DumpCacheFutureTask.java | 32 ++++---- .../persistence/snapshot/dump/DumpMetadata.java | 91 ---------------------- 3 files changed, 48 insertions(+), 121 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index a8b80df7da2..9144f5d86ab 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -142,7 +142,6 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadO import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage; import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpCacheFutureTask; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.DumpMetadata; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -1233,17 +1232,17 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** * @param req Request. - * @param grps0 Cache groups to dump. + * @param grps Cache groups to dump. * @return Create dump future. */ - private IgniteInternalFuture<SnapshotOperationResponse> initLocalDump(SnapshotOperationRequest req, List<Integer> grps0) { + private IgniteInternalFuture<SnapshotOperationResponse> initLocalDump(SnapshotOperationRequest req, List<Integer> grps) { IgniteInternalFuture<?> task0; - List<Integer> grps = grps0.stream().filter(grpId -> cctx.cache().cacheGroup(grpId) != null).collect(Collectors.toList()); + List<Integer> grpIds = grps.stream().filter(grpId -> cctx.cache().cacheGroup(grpId) != null).collect(Collectors.toList()); File dumpDir = snapshotLocalDir(req.snapshotName(), null, locDumpDir); - if (grps.isEmpty()) + if (grpIds.isEmpty()) task0 = new GridFinishedFuture<>(Collections.emptySet()); else { dumpDir.mkdirs(); @@ -1257,7 +1256,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter dumpDir, tmpWorkDir, ioFactory, - grps + grpIds )); } @@ -1269,23 +1268,39 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter .map(n -> cctx.discovery().node(n).consistentId().toString()) .collect(Collectors.toSet()); - // Ignoring Void result fut.result(). + SnapshotFutureTaskResult res = (SnapshotFutureTaskResult)fut.result(); - DumpMetadata meta = new DumpMetadata( - req.requestId(), - cctx.localNode().consistentId().toString(), + SnapshotMetadata meta = new SnapshotMetadata(req.requestId(), req.snapshotName(), - grps, - nodes + cctx.localNode().consistentId().toString(), + pdsSettings.folderName(), + cctx.gridConfig().getDataStorageConfiguration().getPageSize(), + grpIds, + Collections.emptyList(), + nodes, + res.parts(), + null, + null, + false ); + SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, req.groups(), cctx.localNode(), dumpDir, + req.streamerWarning(), true); + + req.meta(meta); + File dmf = new File(dumpDir, dumpMetaFileName(cctx.localNode().consistentId().toString())); storeSnapshotMeta(meta, dmf); log.info("Dump metafile has been created: " + dmf.getAbsolutePath()); - return new SnapshotOperationResponse(null); + try { + return new SnapshotOperationResponse(handlers.invokeAll(SnapshotHandlerType.CREATE, ctx)); + } + catch (IgniteCheckedException e) { + throw F.wrap(e); + } }, snapshotExecutorService()); } @@ -2612,7 +2627,10 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter * @param grps List of cache groups which will be destroyed. */ public void onCacheGroupsStopped(List<Integer> grps) { - for (AbstractSnapshotFutureTask<?> sctx : F.view(locSnpTasks.values(), t -> t instanceof SnapshotFutureTask)) { + Collection<AbstractSnapshotFutureTask<?>> tasks = + F.view(locSnpTasks.values(), t -> t instanceof SnapshotFutureTask || t instanceof DumpCacheFutureTask); + + for (AbstractSnapshotFutureTask<?> sctx : tasks) { Set<Integer> retain = new HashSet<>(grps); retain.retainAll(sctx.affectedCacheGroups()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpCacheFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpCacheFutureTask.java index 3b4b205c9e6..00c77c3ba38 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpCacheFutureTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpCacheFutureTask.java @@ -137,28 +137,28 @@ public class DumpCacheFutureTask extends AbstractSnapshotFutureTask<Void> { )); for (int grp : grps) { - CacheGroupContext grpCtx = cctx.cache().cacheGroup(grp); - - File grpDir = new File( - dumpNodeDir, - (grpCtx.caches().size() > 1 ? CACHE_GRP_DIR_PREFIX : CACHE_DIR_PREFIX) + grpCtx.cacheOrGroupName() - ); + futs.add(CompletableFuture.runAsync(wrapExceptionIfStarted(() -> { + long start = System.currentTimeMillis(); - IgniteUtils.ensureDirectory(grpDir, "dump group directory", null); + CacheGroupContext grpCtx = cctx.cache().cacheGroup(grp); - for (GridCacheContext<?, ?> cacheCtx : grpCtx.caches()) { - CacheConfiguration<?, ?> ccfg = cacheCtx.config(); + log.info("Start group dump [name=" + grpCtx.cacheOrGroupName() + ", id=" + grp + ']'); - cctx.cache().configManager().writeCacheData( - new StoredCacheData(ccfg), - new File(grpDir, cachDataFilename(ccfg)) + File grpDir = new File( + dumpNodeDir, + (grpCtx.caches().size() > 1 ? CACHE_GRP_DIR_PREFIX : CACHE_DIR_PREFIX) + grpCtx.cacheOrGroupName() ); - } - futs.add(CompletableFuture.runAsync(wrapExceptionIfStarted(() -> { - long start = System.currentTimeMillis(); + IgniteUtils.ensureDirectory(grpDir, "dump group directory", null); - log.info("Start group dump [name=" + grpCtx.cacheOrGroupName() + ", id=" + grp + ']'); + for (GridCacheContext<?, ?> cacheCtx : grpCtx.caches()) { + CacheConfiguration<?, ?> ccfg = cacheCtx.config(); + + cctx.cache().configManager().writeCacheData( + new StoredCacheData(ccfg), + new File(grpDir, cachDataFilename(ccfg)) + ); + } try { Thread.sleep(ThreadLocalRandom.current().nextInt(5_000)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpMetadata.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpMetadata.java deleted file mode 100644 index 42b969ce6cc..00000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/DumpMetadata.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.persistence.snapshot.dump; - -import java.io.Serializable; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import org.apache.ignite.internal.util.tostring.GridToStringInclude; - -/** - * - */ -public class DumpMetadata implements Serializable { - /** Serial version uid. */ - private static final long serialVersionUID = 0L; - - /** Unique dump request id. */ - private final UUID rqId; - - /** Snapshot name. */ - @GridToStringInclude - private final String dumpName; - - /** Consistent id of a node to which this metadata relates. */ - @GridToStringInclude - private final String consId; - - /** The list of cache groups ids which were included into dump. */ - @GridToStringInclude - private final List<Integer> grpIds; - - /** The set of affected by dump nodes. */ - @GridToStringInclude - private final Set<String> nodes; - - /** - * @param rqId Unique request id. - * @param dumpName Dump name. - * @param consId Consistent id of a node to which this metadata relates. - * @param grpIds The list of cache groups ids which were included into dump. - * @param nodes The set of affected by dump nodes. - */ - public DumpMetadata(UUID rqId, String dumpName, String consId, List<Integer> grpIds, Set<String> nodes) { - this.rqId = rqId; - this.dumpName = dumpName; - this.consId = consId; - this.grpIds = grpIds; - this.nodes = nodes; - } - - /** */ - public UUID requestId() { - return rqId; - } - - /** */ - public String dumpName() { - return dumpName; - } - - /** */ - public String consistentId() { - return consId; - } - - /** */ - public List<Integer> groups() { - return grpIds; - } - - /** */ - public Set<String> nodes() { - return nodes; - } -}
