This is an automated email from the ASF dual-hosted git repository.
gortiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2904ceb407 Delete database API (#12765)
2904ceb407 is described below
commit 2904ceb4074ee222eeffa7d6806bd5e220f8d8a8
Author: Shounak kulkarni <[email protected]>
AuthorDate: Tue Apr 2 23:10:37 2024 +0500
Delete database API (#12765)
* delete database API
* add field for failed table deletion list along with cause of failure
* remove _tablesInDatabase and _partiallyDeleted from DeleteDatabaseResponse
* set default value for dryRun as true
---
.../resources/PinotDatabaseRestletResource.java | 95 +++++++++++++++++++
.../PinotDatabaseRestletResourceTest.java | 102 +++++++++++++++++++++
.../java/org/apache/pinot/core/auth/Actions.java | 1 +
3 files changed, 198 insertions(+)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResource.java
index ac172bd3e5..e09829292c 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResource.java
@@ -18,23 +18,31 @@
*/
package org.apache.pinot.controller.api.resources;
+import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiKeyAuthDefinition;
import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
+import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.core.auth.Actions;
import org.apache.pinot.core.auth.Authorize;
import org.apache.pinot.core.auth.TargetType;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,4 +67,91 @@ public class PinotDatabaseRestletResource {
public List<String> listDatabaseNames() {
return _pinotHelixResourceManager.getDatabaseNames();
}
+
+ @DELETE
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("/databases/{databaseName}")
+ @Authorize(targetType = TargetType.CLUSTER, action =
Actions.Cluster.DELETE_DATABASE)
+ @ApiOperation(value = "Delete all tables in given database name", notes =
"Delete all tables in given database name")
+ public DeleteDatabaseResponse deleteTablesInDatabase(
+ @ApiParam(value = "Database name", required = true)
@PathParam("databaseName") String databaseName,
+ @ApiParam(value = "Run in dryRun mode initially to know the list of
tables that will be deleted in actual run. "
+ + "No tables will be deleted when dryRun=true", required = true,
defaultValue = "true")
+ @QueryParam("dryRun") boolean dryRun) {
+ List<String> tablesInDatabase =
_pinotHelixResourceManager.getAllTables(databaseName);
+ List<String> deletedTables = new ArrayList<>(tablesInDatabase.size());
+ List<DeletionFailureWrapper> failedTables = new
ArrayList<>(tablesInDatabase.size());
+ if (dryRun) {
+ deletedTables.addAll(tablesInDatabase);
+ } else {
+ for (String table : tablesInDatabase) {
+ boolean isSchemaDeleted = false;
+ try {
+ TableType tableType =
TableNameBuilder.getTableTypeFromTableName(table);
+ String rawTableName = TableNameBuilder.extractRawTableName(table);
+ _pinotHelixResourceManager.deleteSchema(rawTableName);
+ LOGGER.info("Deleted schema: {}", rawTableName);
+ isSchemaDeleted = true;
+ _pinotHelixResourceManager.deleteTable(table, tableType, null);
+ LOGGER.info("Deleted table: {}", table);
+ deletedTables.add(table);
+ } catch (Exception e) {
+ if (isSchemaDeleted) {
+ LOGGER.error("Failed to delete table {}", table);
+ } else {
+ LOGGER.error("Failed to delete table and schema for {}", table);
+ }
+ failedTables.add(new DeletionFailureWrapper(table, e.getMessage()));
+ }
+ }
+ }
+ return new DeleteDatabaseResponse(deletedTables, failedTables, dryRun);
+ }
+}
+
+class DeleteDatabaseResponse {
+ private final List<String> _deletedTables;
+ private final List<DeletionFailureWrapper> _failedTables;
+ private final boolean _dryRun;
+
+ public DeleteDatabaseResponse(List<String> deletedTables,
List<DeletionFailureWrapper> failedTables, boolean dryRun) {
+ _deletedTables = deletedTables;
+ _failedTables = failedTables;
+ _dryRun = dryRun;
+ }
+
+ @JsonProperty("deletedTables")
+ public List<String> getDeletedTables() {
+ return _deletedTables;
+ }
+
+ @JsonProperty("failedTables")
+ public List<DeletionFailureWrapper> getFailedTables() {
+ return _failedTables;
+ }
+
+ @JsonProperty("dryRun")
+ public boolean isDryRun() {
+ return _dryRun;
+ }
+}
+
+class DeletionFailureWrapper {
+ private final String _tableName;
+ private final String _errorMessage;
+
+ public DeletionFailureWrapper(String tableName, String errorMessage) {
+ _tableName = tableName;
+ _errorMessage = errorMessage;
+ }
+
+ @JsonProperty("tableName")
+ public String getTableName() {
+ return _tableName;
+ }
+
+ @JsonProperty("errorMessage")
+ public String getErrorMessage() {
+ return _errorMessage;
+ }
}
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResourceTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResourceTest.java
new file mode 100644
index 0000000000..877c65f96b
--- /dev/null
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotDatabaseRestletResourceTest.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api.resources;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+
+public class PinotDatabaseRestletResourceTest {
+ private static final String DATABASE = "db";
+ private static final List<String> TABLES = Lists.newArrayList("a_REALTIME",
"b_OFFLINE", "c_REALTIME", "d_OFFLINE");
+
+ @Mock
+ PinotHelixResourceManager _resourceManager;
+ @InjectMocks
+ PinotDatabaseRestletResource _resource;
+
+ @BeforeMethod
+ public void setup() {
+ MockitoAnnotations.openMocks(this);
+ when(_resourceManager.getAllTables(DATABASE)).thenReturn(TABLES);
+ doNothing().when(_resourceManager).deleteTable(anyString(),
any(TableType.class), any());
+ when(_resourceManager.deleteSchema(anyString())).thenReturn(true);
+ }
+
+ @Test
+ public void successfulDatabaseDeletionDryRunTest() {
+ successfulDatabaseDeletionCheck(true);
+ }
+
+ @Test
+ public void successfulDatabaseDeletionTest() {
+ successfulDatabaseDeletionCheck(false);
+ }
+
+ private void successfulDatabaseDeletionCheck(boolean dryRun) {
+ DeleteDatabaseResponse response =
_resource.deleteTablesInDatabase(DATABASE, dryRun);
+ assertEquals(response.isDryRun(), dryRun);
+ assertTrue(response.getFailedTables().isEmpty());
+ assertEquals(response.getDeletedTables(), TABLES);
+ }
+
+ @Test
+ public void partialDatabaseDeletionWithDeleteTableFailureTest() {
+ int failureTableIdx = TABLES.size() / 2;
+ doThrow(new RuntimeException()).when(_resourceManager)
+ .deleteTable(TABLES.get(failureTableIdx), TableType.REALTIME, null);
+ partialDatabaseDeletionCheck(failureTableIdx);
+ }
+
+ @Test
+ public void partialDatabaseDeletionWithDeleteSchemaFailureTest() {
+ int failureSchemaIdx = TABLES.size() / 2;
+ doThrow(new RuntimeException()).when(_resourceManager)
+
.deleteSchema(TableNameBuilder.extractRawTableName(TABLES.get(failureSchemaIdx)));
+ partialDatabaseDeletionCheck(failureSchemaIdx);
+ }
+
+ private void partialDatabaseDeletionCheck(int idx) {
+ DeleteDatabaseResponse response =
_resource.deleteTablesInDatabase(DATABASE, false);
+ List<String> resultList = new ArrayList<>(TABLES);
+ String failedTable = resultList.remove(idx);
+ assertFalse(response.isDryRun());
+ assertEquals(response.getFailedTables().size(), 1);
+ assertEquals(response.getFailedTables().get(0).getTableName(),
failedTable);
+ assertEquals(response.getDeletedTables(), resultList);
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
index 3a3e4ac593..3793d773a6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
@@ -40,6 +40,7 @@ public class Actions {
public static final String DELETE_TENANT = "DeleteTenant";
public static final String DELETE_USER = "DeleteUser";
public static final String DELETE_ZNODE = "DeleteZnode";
+ public static final String DELETE_DATABASE = "DeleteDatabase";
public static final String ESTIMATE_UPSERT_MEMORY = "EstimateUpsertMemory";
public static final String EXECUTE_TASK = "ExecuteTask";
public static final String GET_ADMIN_INFO = "GetAdminInfo";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
