IGNITE-29050 Introduce snapshot deletion command

docs/_docs/snapshots/snapshots.adoc

@@ -287,6 +287,50 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach
 control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1
 ----
 
+== Deleting Snapshot
+
+You can delete a snapshot using the `control.sh|bat` script.
+
+The deletion is performed on all *online* server nodes of the cluster.
+[NOTE]
+====
+The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted.
+====
+
+[tabs]
+--
+tab:Unix[]
+[source,shell]
+----
+# Delete the snapshot "snapshot_09062021".
+control.sh --snapshot delete snapshot_09062021
+
+# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder.
+control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots
+----
+
+tab:Windows[]
+[source,shell]
+----
+# Delete the snapshot "snapshot_09062021".
+control.bat --snapshot delete snapshot_09062021
+
+# Delete the snapshot "snapshot_09062021" located in the "C:\tmp\ignite\snapshots" folder.
+control.bat --snapshot delete snapshot_09062021 --src C:\tmp\ignite\snapshots
+----
+--
+
+=== Delete operation limitations
+
+The delete operation is subject to the following limitations:
+
+* The deletion is rejected if any snapshot operation (create, restore, check, delete) is active for the snapshot.
+* The operation requires the snapshot administration permissions via `IgniteSecurity` (if configured).
+* The operation cannot be undone and the deleted snapshot cannot be restored. The command prompts for a confirmation.
+* Before deletion, no validation is performed on the snapshot other than locating and reading its metadata.
+If the metadata isn't found or cannot be read, snapshot isn't deleted.
+* The deletion operation is independent of the status of the snapshot operation and cancellation requests.
+
 == Getting Snapshot Operation Status
 
 The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface:

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java

@@ -99,7 +99,7 @@ public static List<RelHint> nonInheritedRelHints(RelNode rel) {
 
     /**
      * @return Distinct hints within {@code hints} filtered with {@code hintDefs}, {@link HintOptionsChecker} and
-     * removed inherit pathes.
+     * removed inherit paths.
      * @see HintOptionsChecker
      * @see RelHint#inheritPath
      */

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java

@@ -134,7 +134,7 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner, IgniteL
             rel = planner.trimUnusedFields(root.withRel(rel)).rel;
 
             // The following pushed down project can erase top-level hints. We store them to reassign hints for join nodes.
-            // Clear the inherit pathes to consider the hints as not propogated ones.
+            // Clear the inherit paths to consider the hints as not propagated ones.
             List<RelHint> topHints = HintUtils.allRelHints(rel).stream().map(h -> h.inheritPath.isEmpty()
                 ? h
                 : h.copy(Collections.emptyList())).collect(Collectors.toList());

modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java

@@ -81,6 +81,7 @@
 import org.apache.ignite.internal.management.performancestatistics.PerformanceStatisticsCommand;
 import org.apache.ignite.internal.management.property.PropertyCommand;
 import org.apache.ignite.internal.management.snapshot.SnapshotCommand;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand;
 import org.apache.ignite.internal.management.snapshot.SnapshotRestoreCommand;
 import org.apache.ignite.internal.management.tx.TxCommand;
 import org.apache.ignite.internal.management.tx.TxCommandArg;
@@ -529,6 +530,8 @@ else if (cmd.getClass() == EncryptionChangeCacheKeyCommand.class)
             cmdText = F.concat(cmdText, "cacheGroup1");
         else if (cmd.getClass() == SnapshotRestoreCommand.class)
             cmdText = F.concat(cmdText, "snp1");
+        else if (cmd.getClass() == SnapshotDeleteCommand.class)
+            cmdText = F.concat(cmdText, "snp1");
         else if (cmd.getClass() == MetaUpdateCommand.class)
             return;
         else if (cmd.getClass() == MetaRemoveCommand.class)

modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java

@@ -27,6 +27,7 @@
 import org.apache.ignite.util.GridCommandHandlerCheckpointTest;
 import org.apache.ignite.util.GridCommandHandlerClusterByClassTest;
 import org.apache.ignite.util.GridCommandHandlerClusterByClassWithSSLTest;
+import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest;
 import org.apache.ignite.util.GridCommandHandlerIncompatibleSslConfigTest;
 import org.apache.ignite.util.GridCommandHandlerIndexingCheckSizeTest;
 import org.apache.ignite.util.GridCommandHandlerIndexingClusterByClassTest;
@@ -74,6 +75,7 @@
     GridCommandHandlerCheckIndexesInlineSizeTest.class,
     GridCommandHandlerMetadataTest.class,
     GridCommandHandlerCheckIncrementalSnapshotTest.class,
+    GridCommandHandlerDeleteSnapshotTest.class,
     GridCommandHandlerLegacyClientTest.class,
 
     KillCommandsControlShTest.class,

modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for control utility.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+    GridCommandHandlerDeleteSnapshotTest.class
+})
+public class IgniteControlUtilityTestSuite6 {
+}

modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java

@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import static java.nio.file.Files.newDirectoryStream;
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.junit.Assume.assumeTrue;
+
+/** Test for the command '--snapshot delete'. */
+public class GridCommandHandlerDeleteSnapshotTest extends GridCommandHandlerAbstractTest {
+    /** Flag to add an extra server node after the cluster start and the snapshot creation. */
+    @Parameter(1)
+    public boolean extraServerNode;
+
+    /** */
+    @Parameter(2)
+    public boolean incremental;
+
+    /** */
+    @Parameter(3)
+    public boolean changeBaseline;
+
+    /** */
+    @Parameter(4)
+    public boolean customPath;
+
+    /** */
+    @Parameter(5)
+    public boolean separatedWorkDir;
+
+    /** */
+    @Parameters(name = "client={0},useExtraNode={1},inc={2},chBaseln={3},cstSnpPath={4},ownWorkDir={5}")
+    public static Collection<?> parameters() {
+        return GridTestUtils.cartesianProduct(
+            commandHandlers(),
+            F.asList(false, true), // Use or not extra server node after the cluster start and the snapshot creation;
+            F.asList(false, true), // Add incremental snapshot;
+            F.asList(false, true), // Change baseline;
+            F.asList(false, true), // Use custom snapshot path;
+            F.asList(false, true) // Separated (own) work directory.
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        /** Handy if test running is interrupted and {@link #afterTest()} isn't invoked. */
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void cleanPersistenceDir() throws Exception {
+        super.cleanPersistenceDir();
+
+        // Also cleans separated snapshot working directories and custom snapshot paths.
+        try (DirectoryStream<Path> files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) {
+            for (Path path : files)
+                U.delete(path);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (separatedWorkDir)
+            cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath());
+
+        return cfg;
+    }
+
+    /** */
+    @Test
+    public void testSnapshotDelete() throws Exception {
+        // A custom snapshot path actually puts snapshots in a shared directory. This skews the results when dedicated
+        // work directories are set.
+        assumeTrue(!customPath || !separatedWorkDir);
+
+        int entriesCnt = 100;
+        int initNodes = 3;
+
+        walCompactionEnabled(incremental);
+
+        IgniteEx ig = (IgniteEx)startGridsMultiThreaded(initNodes);
+
+        if (changeBaseline) {
+            ig.cluster().baselineAutoAdjustEnabled(false);
+
+            ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
+        }
+
+        ig.cluster().state(ACTIVE);
+
+        createCacheAndPreload(ig, entriesCnt);
+
+        File cstSnpsRoot = customPath
+            ? new File(grid(0).context().pdsFolderResolver().fileTree().snapshotsRoot(), "ex_snapshots")
+            : null;
+        File snpDir = new File(customPath ? cstSnpsRoot : ig.context().pdsFolderResolver().fileTree().snapshotsRoot(), "testSnapshot");
+
+        snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, false, false)
+            .get(getTestTimeout());
+
+        if (incremental) {
+            for (int i = 0; i < 3; ++i) {
+                int dataIdx = entriesCnt + entriesCnt / 4 * i;
+
+                try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
+                    for (int d = dataIdx; d < dataIdx + entriesCnt / 4; ++d)
+                        streamer.addData(d, d);
+                }
+
+                snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, true, false)
+                    .get(getTestTimeout());
+            }
+        }
+
+        String baselineGone = null;
+
+        // Optionally restarts with the same servers number, but changed baseline. The snapshot is kept on the same
+        // previous nodes independently of the baseline.
+        if (changeBaseline) {
+            baselineGone = grid(initNodes - 1).name();
+
+            ig.destroyCache(DEFAULT_CACHE_NAME);
+            awaitPartitionMapExchange();
+
+            stopAllGrids();
+
+            ig = (IgniteEx)startGridsMultiThreaded(initNodes - 1);
+
+            ig.cluster().setBaselineTopology(ig.cluster().topologyVersion());
+
+            startGrid(initNodes - 1);
+
+            assertEquals(initNodes - 1, ig.cluster().currentBaselineTopology().size());
+            assertEquals(initNodes, ig.cluster().nodes().size());
+        }
+
+        // Optionally adds extra server.
+        if (extraServerNode)
+            startGrid(initNodes);
+
+        injectTestSystemOut();
+
+        // Tests missing snapshot deletion.
+        if (customPath) {
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "--src",
+                cstSnpsRoot.getAbsolutePath(), "wrongSnapshot"));
+
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "--src",
+                cstSnpsRoot.getAbsolutePath() + "_wrongPath", "testSnapshot"));
+        }
+        else
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "wrongSnapshot"));
+
+        String out = testOut.toString();
+
+        assertFalse(out.contains(SnapshotDeleteCommand.REMOVED_PREF));
+        assertFalse(out.contains(SnapshotDeleteCommand.NODE_NOT_FOUND_PREF));
+        assertTrue(out.contains(SnapshotDeleteCommand.NOT_FOUND));
+
+        testOut.reset();
+        assertTrue(testOut.toString().isEmpty());
+
+        if (customPath) {
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "--src",
+                cstSnpsRoot.getAbsolutePath(), "testSnapshot"));
+        }
+        else
+            assertEquals(EXIT_CODE_OK, execute(newCommandHandler(createTestLogger()), "--snapshot", "delete", "testSnapshot"));
+
+        out = testOut.toString();
+
+        if (changeBaseline)
+            assertTrue(out.contains(SnapshotDeleteCommand.MISSING_BASELINES + baselineGone));
+
+        if (separatedWorkDir) {
+            // When the nodes use own separated work directory, we expect a strict result.
+            assertTrue(out.contains(SnapshotDeleteCommand.REMOVED_PREF));
+
+            if (extraServerNode)
+                assertTrue(out.contains(SnapshotDeleteCommand.NODE_NOT_FOUND_PREF));
+        }
+        else {
+            // When nodes use a shared work directory, there is a race for the delete operation. One node can get faster
+            // than others and remove snapshot completely quickly. The others might not find snapshot files. We can be
+            // only sure that at least one node removes snapshot.
+            assertTrue(out.contains(SnapshotDeleteCommand.REMOVED_PREF) || out.contains(SnapshotDeleteCommand.UNSURE_DELETION_PREF));
+        }
+
+        assertFalse(out.contains("Snapshot not found on current server nodes"));
+
+        assertTrue(waitForCondition(() -> !snpDir.exists(), getTestTimeout()));
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java

@@ -162,6 +162,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckPartitionHashesResponse;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcessRequest;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckResponse;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult;
@@ -436,6 +438,8 @@ public CoreMessagesProvider() {
         register(SnapshotFilesFailureMessage.class);
         register(IncrementalSnapshotVerifyResult.class);
         register(IncrementalSnapshotAwareMessage.class);
+        register(SnapshotDeleteRequest.class);
+        register(SnapshotDeleteResponse.class);
 
         // [6300 - 6400]: Services messages. Most of them originally come from Discovery.
         msgIdx = 6300;