zstan commented on code in PR #13577: URL: https://github.com/apache/ignite/pull/13577#discussion_r4144520181
########## modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java: ########## @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Cluster snapshot delete distributed process request. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteRequest implements Message { + /** Request ID. */ + @Order(0) + UUID reqId; + + /** Snapshot name. */ + @Order(1) + String snpName; + + /** Snapshot directory path. */ + @Order(2) + @Nullable String snpPath; + + /** Resolved canonical sanpshot full path (with the snapshot name). Transient */ Review Comment: ```suggestion /** Resolved canonical snapshot full path (with the snapshot name). Transient */ ``` Transient ??? ########## modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java: ########## @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.lang.IgniteReducer; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT; + +/** + * Distributed process to delete a cluster snapshot. The operation is rejected if any concurrent snapshot operation is + * active. + */ +public class SnapshotDeleteProcess { + /** Reject operation messages. */ + private static final String OP_REJECT_MSG = "Snapshot deletion was rejected. "; + + /** */ + public static final String OP_REJECT_FEATURE_MSG = OP_REJECT_MSG + "The snapshot deletion feature isn't activated yet."; + + /** */ + private static final String CONCURRENT_OP_PREF = "Snapshot with the same name is being "; + + /** */ + public static final String BEING_CREATED_PREF = CONCURRENT_OP_PREF + "created "; + + /** */ + public static final String BEING_RESTORED_PREF = CONCURRENT_OP_PREF + "restored "; + + /** */ + public static final String BEING_CHECKED_PREF = CONCURRENT_OP_PREF + "checked "; + + /** Kernal context. */ + private final GridKernalContext kctx; + + /** Logger. */ + private final IgniteLogger log; + + /** */ + private volatile boolean interrupted; + + /** Cluster-wide operation futures per request id on certain node. */ + private final Map<UUID, GridFutureAdapter<SnapshotDeleteProcessResult>> clusterOpFuts = new ConcurrentHashMap<>(); + + /** Process requests per snapshot name on each server node. */ + private final Set<SnapshotDeleteRequest> requests = ConcurrentHashMap.newKeySet(); + + /** The distributed process. */ + private final DistributedProcess<SnapshotDeleteRequest, SnapshotDeleteResponse> distrProc; + + /** + * @param ctx Kernal context. + */ + public SnapshotDeleteProcess(GridKernalContext ctx) { + kctx = ctx; + + log = ctx.log(getClass()); + + distrProc = new DistributedProcess<>(ctx, DELETE_SNAPSHOT, this::deletePhase, this::reducePhase); + } + + /** + * Starts the cluster snapshot delete process. + * + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path (optional). + * @return Future that will be completed when the snapshot is deleted. + */ + public IgniteFuture<SnapshotDeleteProcessResult> start(String snpName, @Nullable String snpPath) { + GridFutureAdapter<SnapshotDeleteProcessResult> clusterOpFut = new GridFutureAdapter<>(); + + if (!kctx.rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE)) { + clusterOpFut.onDone(new IgniteIllegalStateException(OP_REJECT_FEATURE_MSG)); + + return new IgniteFutureImpl<>(clusterOpFut); + } + + UUID reqId = UUID.randomUUID(); + + clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId)); + + try { + if (interrupted || kctx.isStopping()) + throw new NodeStoppingException("Failed to start snapshot delete process: node is stopping."); + + clusterOpFuts.put(reqId, clusterOpFut); + + SnapshotDeleteRequest req = new SnapshotDeleteRequest(reqId, snpName, snpPath); + + distrProc.start(reqId, req); + } + catch (Throwable t) { + log.error("Failed to start distributed delete snapshot process [snpName=" + snpName + ", snpPath=" + snpPath + ']', t); + + clusterOpFut.onDone(t); + } + + return new IgniteFutureImpl<>(clusterOpFut); + } + + /** */ + private IgniteInternalFuture<SnapshotDeleteResponse> deletePhase(UUID ignored, SnapshotDeleteRequest req) { + if (interrupted || kctx.isStopping()) { + return new GridFinishedFuture<>(new NodeStoppingException(OP_REJECT_MSG + + " Node is stopping [req=" + req + ']')); + } + + if (kctx.cluster().get().localNode().isClient()) + return new GridFinishedFuture<>(new SnapshotDeleteResponse()); + + kctx.security().authorize(ADMIN_SNAPSHOT); + + IgniteSnapshotManager snpMgr = kctx.cache().context().snapshotMgr(); + + SnapshotOperationRequest curCreateRq = snpMgr.currentCreateRequest(); + + if (curCreateRq != null && curCreateRq.snpName.equalsIgnoreCase(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_CREATED_PREF + + "[req=" + req + ']')); + } + + if (snpMgr.isRestoring(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_RESTORED_PREF + + "[req=" + req + ']')); + } + + if (snpMgr.isSnapshotChecking(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_CHECKED_PREF + + "[req=" + req + ']')); + } + + try { + req.resolvedFullPath = resolveFullPath(req.snpName, req.snpPath); + + File rootPath = req.resolvedFullPath.getParentFile(); + + if (!requests.add(req)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Deletion of the snapshot has already " + + "started [req=" + req + ']')); + } + + SnapshotFileTree snpFiles = new SnapshotFileTree(kctx, req.snpName, rootPath.getAbsolutePath()); + + // We need to find and read snapshot metas to ensure the content is a snapshot. Also, the metas contain + // initial cluster topology and actual snapshot folder names. + List<SnapshotMetadata> locMetas = kctx.cache().context().snapshotMgr().readSnapshotMetadatas(snpFiles, false); + + if (locMetas.isEmpty()) { + requests.remove(req); + + log.warning("Snapshot deletion won't process, no snapshot metadata found [req=" + req + ']'); + + return new GridFinishedFuture<>(new SnapshotDeleteResponse(SnapshotDeleteResponse.DeleteStatus.NOT_FOUND, null)); + } + + // Future to delete snapshot contents according to snapshot metadatas. + GridCompoundFuture<SnapshotDeleteResponse, SnapshotDeleteResponse> resultFut = + new GridCompoundFuture<>(new MetaFuturesReducer()); + + resultFut.listen(fut -> requests.remove(req)); + + for (SnapshotMetadata meta : locMetas) { + GridFutureAdapter<SnapshotDeleteResponse> perMetaFut = new GridFutureAdapter<>(); + + kctx.pools().getSnapshotExecutorService().submit(() -> { + try { + // Read file tree of the snapshot. + var byMetaSft = new SnapshotFileTree( + kctx.config(), + kctx.pdsFolderResolver().fileTree(), + req.snpName, + req.snpPath == null ? null : rootPath.getAbsolutePath(), + meta.folderName(), + meta.consId + ); + + T2<Boolean, Boolean> deleted = snpMgr.deleteLocalSnapshot(byMetaSft); + + SnapshotDeleteResponse.DeleteStatus status; + + // If found. + if (deleted.get2()) { + if (deleted.get1() && log.isInfoEnabled()) + log.info("Snapshot successfully deleted [req=" + req + ']'); + else if (!deleted.get1()) + log.warning("Snapshot deleted not completely [req=" + req + ']'); + + status = deleted.get1() + ? SnapshotDeleteResponse.DeleteStatus.DELETED + : SnapshotDeleteResponse.DeleteStatus.PARTLY; + } + else { + if (log.isInfoEnabled()) + log.info("Snapshot not found to delete [req=" + req + ']'); + + status = SnapshotDeleteResponse.DeleteStatus.NOT_FOUND; + } + + perMetaFut.onDone(new SnapshotDeleteResponse(status, meta.bltNodes)); + } + catch (Throwable e) { + perMetaFut.onDone(e); + } + }); + + resultFut.add(perMetaFut); + } + + resultFut.markInitialized(); + + if (log.isInfoEnabled()) + log.info("Deletion of snapshot initialized [req=" + req + ']'); + + return resultFut; + } + catch (Throwable t) { + requests.remove(req); + + log.warning("An error occurred during snapshot deletion [req=" + req + ']', t); + + return new GridFinishedFuture<>(t); + } + } + + /** */ + private File resolveFullPath(String name, @Nullable String path) throws IOException { + return new File(path == null ? kctx.pdsFolderResolver().fileTree().snapshotsRoot().getAbsolutePath() : path, name) + .getCanonicalFile(); + } + + /** */ + private void reducePhase(UUID reqId, Map<UUID, SnapshotDeleteResponse> results, Map<UUID, Throwable> errors) { + GridFutureAdapter<SnapshotDeleteProcessResult> clusterOpFut = clusterOpFuts.get(reqId); + + if (clusterOpFut == null) + return; + + assert clusterOpFut != null; + + try { + Map.Entry<UUID, Throwable> errP = F.isEmpty(errors) ? null : F.first(errors.entrySet()); + + if (errP != null) { + log.warning("Snapshot deletion finished with an error [reqId=" + reqId + ", nodeId=" + + errP.getKey() + ", err='" + errP.getValue().getMessage() + "']", errP.getValue()); + + clusterOpFut.onDone(errP.getValue()); + + return; + } + + Map<UUID, String> completedNodes = U.newHashMap(results.size()); + Map<UUID, String> uncompletedNodes = U.newHashMap(results.size()); + Map<UUID, String> emptyNodes = U.newHashMap(results.size()); + var snpNodes = new HashSet<String>(); + + results.forEach((nodeId, nodeRes) -> { + if (!F.isEmpty(nodeRes.nodeIds)) + snpNodes.addAll(nodeRes.nodeIds); + + if (nodeRes.status != null) { + switch (nodeRes.status) { + case NOT_FOUND: + emptyNodes.put(nodeId, consistentId(nodeId)); + break; + case DELETED: + completedNodes.put(nodeId, consistentId(nodeId)); + break; + case PARTLY: + uncompletedNodes.put(nodeId, consistentId(nodeId)); + break; + default: + throw new IgniteIllegalStateException("Unknown snapshot deletion node result, [nodeRes=" + + nodeRes + ", nodeId=" + nodeId + ']'); + } + } + }); + + Collection<ClusterNode> curBaseline = kctx.discovery().discoCache().aliveBaselineNodes(); + + if (!F.isEmpty(curBaseline)) + curBaseline.stream().map(bn -> bn.consistentId().toString()).toList().forEach(snpNodes::remove); + + clusterOpFut.onDone(new SnapshotDeleteProcessResult(completedNodes, uncompletedNodes, emptyNodes, snpNodes)); + } + catch (Throwable t) { + clusterOpFut.onDone(t); + } + } + + /** */ + private String consistentId(UUID nodeId) { + ClusterNode node = kctx.discovery().node(nodeId); + + if (node == null) + node = kctx.discovery().historicalNode(nodeId); + + return node == null ? "" : node.consistentId().toString(); + } + + /** */ + public boolean isDeleting(String snpName, @Nullable String snpPath) { + var rq = new SnapshotDeleteRequest(null, snpName, snpPath); + + try { + rq.resolvedFullPath = resolveFullPath(snpName, rq.snpPath); + } + catch (IOException ignored) { + return false; + } + + return requests.contains(rq); + } + + /** + * @param err The interrupt reason. + */ + void interrupt(Throwable err) { + interrupted = true; + + clusterOpFuts.forEach((reqId, clusterOpFut) -> clusterOpFut.onDone(err)); + } + + /** */ + private static class MetaFuturesReducer implements IgniteReducer<SnapshotDeleteResponse, SnapshotDeleteResponse> { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** */ + private @Nullable SnapshotDeleteResponse.DeleteStatus status; + + /** */ + private final Collection<String> nodeIds = new HashSet<>(); + + /** {@inheritDoc} */ + @Override public boolean collect(SnapshotDeleteResponse res) { + assert res != null; + + synchronized (this) { Review Comment: redundant sync or clarify ? ########## docs/_docs/snapshots/snapshots.adoc: ########## @@ -287,6 +287,49 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1 ---- +== Deleting Snapshot + +You can delete a snapshot using the `control.sh|bat` script. + +The deletion is performed on all *online* server nodes of the cluster. +[NOTE] +==== +The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted. +==== + +[tabs] +-- +tab:Unix[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.sh --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder. +control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots +---- + +tab:Windows[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.bat --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "C:\tmp\ignite\snapshots" folder. +control.bat --snapshot delete snapshot_09062021 --src C:\tmp\ignite\snapshots +---- +-- + +=== Delete operation limitations + +The delete operation is subject to the following limitations: + +* The deletion is rejected if any snapshot operation (create, restore, check, delete) is active for the snapshot. +* The operation requires the snapshot administration permissions via `IgniteSecurity` (if configured). +* The operation cannot be undone and the deleted snapshot cannot be restored. The command prompts for a confirmation. +* Before deletion, no validation is performed on the snapshot other than locating and reading its metadata. If the metadata isn't found. Review Comment: WHATTTT ??? ########## modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java: ########## @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Cluster snapshot delete distributed process request. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteRequest implements Message { + /** Request ID. */ + @Order(0) + UUID reqId; + + /** Snapshot name. */ + @Order(1) + String snpName; + + /** Snapshot directory path. */ + @Order(2) + @Nullable String snpPath; + + /** Resolved canonical sanpshot full path (with the snapshot name). Transient */ + @GridToStringExclude + @Nullable File resolvedFullPath; Review Comment: `resolvedFullPath` - but is has no setters ? Why do you need it inside of request ? This is very harmful logic, seems all you need is to remove it from here and change: ``` public class SnapshotDeleteRequest implements Message { /** Request ID. */ @Order(0) UUID reqId; /** Snapshot name. */ @Order(1) String snpName; /** Snapshot directory path. */ @Order(2) @Nullable String snpPath; /** Resolved canonical sanpshot full path (with the snapshot name). Transient */ /** Default constructor for {@link MessageFactory}. */ public SnapshotDeleteRequest() { // No-op. } /** * @param reqId Request ID. * @param snpName Snapshot name. * @param snpPath Snapshot directory path. */ SnapshotDeleteRequest(UUID reqId, String snpName, @Nullable String snpPath) { this.reqId = reqId; this.snpName = snpName; this.snpPath = snpPath; } /** {@inheritDoc} */ @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; SnapshotDeleteRequest other = (SnapshotDeleteRequest)o; return Objects.equals(snpPath, other.snpPath) && Objects.equals(snpName, other.snpName); } /** {@inheritDoc} */ @Override public int hashCode() { return Objects.hash(snpPath, snpName); } ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
