This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 003b2a8256 Check EV not exist before allowing creating the table
(#10593)
003b2a8256 is described below
commit 003b2a825623da54781cefb9e488e372433820f5
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Apr 13 14:32:49 2023 -0700
Check EV not exist before allowing creating the table (#10593)
---
.../api/resources/PinotSchemaRestletResource.java | 25 +++--
.../api/resources/TableConfigsRestletResource.java | 11 +-
.../helix/core/PinotHelixResourceManager.java | 31 ++++--
.../api/PinotSegmentRestletResourceTest.java | 112 +++++++++------------
.../pinot/controller/helix/ControllerTest.java | 32 ++++--
.../pinot/controller/helix/TableCacheTest.java | 2 +-
.../PinotHelixResourceManagerStatelessTest.java | 7 ++
.../tests/SegmentUploadIntegrationTest.java | 5 +-
8 files changed, 127 insertions(+), 98 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
index 66d26c1264..cbc25f5ab9 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
@@ -70,6 +70,7 @@ import org.apache.pinot.segment.local.utils.SchemaUtils;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.glassfish.grizzly.http.server.Request;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
@@ -450,20 +451,28 @@ public class PinotSchemaRestletResource {
}
// If the schema is associated with a table, we should not delete it.
- List<String> tableNames =
_pinotHelixResourceManager.getAllRealtimeTables();
- for (String tableName : tableNames) {
- TableConfig config =
_pinotHelixResourceManager.getRealtimeTableConfig(tableName);
- String tableSchema = config.getValidationConfig().getSchemaName();
-
- if (schemaName.equals(tableSchema)) {
+ // TODO: Check OFFLINE tables as well. There are 2 side effects:
+ // - Increases ZK read when there are lots of OFFLINE tables
+ // - Behavior change since we don't allow deleting schema for
OFFLINE tables
+ List<String> realtimeTables =
_pinotHelixResourceManager.getAllRealtimeTables();
+ for (String realtimeTableName : realtimeTables) {
+ if
(schemaName.equals(TableNameBuilder.extractRawTableName(realtimeTableName))) {
throw new ControllerApplicationException(LOGGER,
- String.format("Cannot delete schema %s, as it is associated with
table %s", schemaName, tableName),
+ String.format("Cannot delete schema %s, as it is associated with
table %s", schemaName, realtimeTableName),
Response.Status.CONFLICT);
}
+ TableConfig tableConfig =
_pinotHelixResourceManager.getTableConfig(realtimeTableName);
+ if (tableConfig != null) {
+ if
(schemaName.equals(tableConfig.getValidationConfig().getSchemaName())) {
+ throw new ControllerApplicationException(LOGGER,
+ String.format("Cannot delete schema %s, as it is associated with
table %s", schemaName,
+ realtimeTableName), Response.Status.CONFLICT);
+ }
+ }
}
LOGGER.info("Trying to delete schema {}", schemaName);
- if (_pinotHelixResourceManager.deleteSchema(schema)) {
+ if (_pinotHelixResourceManager.deleteSchema(schemaName)) {
LOGGER.info("Notifying metadata event for deleting schema: {}",
schemaName);
_metadataEventNotifierFactory.create().notifyOnSchemaEvents(schema,
SchemaEventType.DELETE);
LOGGER.info("Success: Deleted schema {}", schemaName);
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
index 336aa3472e..6e1989339c 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
@@ -227,7 +227,7 @@ public class TableConfigsRestletResource {
// Invoke delete on tables whether they exist or not, to account for
metadata/segments etc.
_pinotHelixResourceManager.deleteRealtimeTable(rawTableName);
_pinotHelixResourceManager.deleteOfflineTable(rawTableName);
- _pinotHelixResourceManager.deleteSchema(schema);
+ _pinotHelixResourceManager.deleteSchema(schema.getSchemaName());
throw e;
}
@@ -271,12 +271,9 @@ public class TableConfigsRestletResource {
LOGGER.info("Deleted realtime table: {}", tableName);
_pinotHelixResourceManager.deleteOfflineTable(tableName);
LOGGER.info("Deleted offline table: {}", tableName);
- Schema schema = _pinotHelixResourceManager.getSchema(tableName);
- if (schema != null) {
- _pinotHelixResourceManager.deleteSchema(schema);
- LOGGER.info("Deleted schema: {}", tableName);
- }
- if (tableExists || schema != null) {
+ boolean schemaExists =
_pinotHelixResourceManager.deleteSchema(tableName);
+ LOGGER.info("Deleted schema: {}", tableName);
+ if (tableExists || schemaExists) {
return new SuccessResponse("Deleted TableConfigs: " + tableName);
} else {
return new SuccessResponse(
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 68008786a6..c8eb5c9df4 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1384,20 +1384,29 @@ public class PinotHelixResourceManager {
* @param schema The schema to be deleted.
* @return True on success, false otherwise.
*/
+ @Deprecated
public boolean deleteSchema(Schema schema) {
if (schema != null) {
- String schemaName = schema.getSchemaName();
- LOGGER.info("Deleting schema: {}", schemaName);
- String propertyStorePath =
ZKMetadataProvider.constructPropertyStorePathForSchema(schemaName);
- if (_propertyStore.exists(propertyStorePath, AccessOption.PERSISTENT)) {
- _propertyStore.remove(propertyStorePath, AccessOption.PERSISTENT);
- LOGGER.info("Deleted schema: {}", schemaName);
- return true;
- }
+ deleteSchema(schema.getSchemaName());
}
return false;
}
+ /**
+ * Deletes the given schema. Returns {@code true} when schema exists, {@code
false} when schema does not exist.
+ */
+ public boolean deleteSchema(String schemaName) {
+ LOGGER.info("Deleting schema: {}", schemaName);
+ String propertyStorePath =
ZKMetadataProvider.constructPropertyStorePathForSchema(schemaName);
+ if (_propertyStore.exists(propertyStorePath, AccessOption.PERSISTENT)) {
+ _propertyStore.remove(propertyStorePath, AccessOption.PERSISTENT);
+ LOGGER.info("Deleted schema: {}", schemaName);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
@Nullable
public Schema getSchema(String schemaName) {
return ZKMetadataProvider.getSchema(_propertyStore, schemaName);
@@ -1472,6 +1481,12 @@ public class PinotHelixResourceManager {
+ " already exists. If this is unexpected, try deleting the table to
remove all metadata associated"
+ " with it.");
}
+ if (_helixAdmin.getResourceExternalView(_helixClusterName,
tableNameWithType) != null) {
+ throw new TableAlreadyExistsException("External view for " +
tableNameWithType
+ + " still exists. If the table is just deleted, please wait for the
clean up to finish before recreating it. "
+ + "If the external view is not removed after a long time, try
restarting the servers showing up in the "
+ + "external view");
+ }
validateTableTenantConfig(tableConfig);
TableType tableType = tableConfig.getTableType();
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentRestletResourceTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentRestletResourceTest.java
index d090316031..181fe6ef44 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentRestletResourceTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotSegmentRestletResourceTest.java
@@ -20,15 +20,18 @@ package org.apache.pinot.controller.api;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.pinot.controller.helix.ControllerTest;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.utils.SegmentMetadataMockUtils;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.ControllerRequestURLBuilder;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.testng.annotations.AfterClass;
@@ -36,14 +39,13 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertThrows;
import static org.testng.Assert.assertTrue;
public class PinotSegmentRestletResourceTest {
private static final ControllerTest TEST_INSTANCE =
ControllerTest.getInstance();
- private static final String TABLE_NAME =
"pinotSegmentRestletResourceTestTable";
- private static final String TABLE_NAME_OFFLINE = TABLE_NAME + "_OFFLINE";
@BeforeClass
public void setUp()
@@ -55,45 +57,41 @@ public class PinotSegmentRestletResourceTest {
public void testListSegmentLineage()
throws Exception {
// Adding table
+ String rawTableName = "lineageTestTable";
+ String offlineTableName =
TableNameBuilder.OFFLINE.tableNameWithType(rawTableName);
TableConfig tableConfig =
- new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setNumReplicas(1).build();
- TEST_INSTANCE.getHelixResourceManager().addTable(tableConfig);
-
- // Wait for the table addition
- while
(!TEST_INSTANCE.getHelixResourceManager().hasOfflineTable(TABLE_NAME)) {
- Thread.sleep(100);
- }
-
- Map<String, SegmentMetadata> segmentMetadataTable = new HashMap<>();
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(rawTableName).setNumReplicas(1).build();
+ PinotHelixResourceManager resourceManager =
TEST_INSTANCE.getHelixResourceManager();
+ resourceManager.addTable(tableConfig);
// Upload Segments
+ Map<String, SegmentMetadata> segmentMetadataTable = new HashMap<>();
for (int i = 0; i < 4; i++) {
- SegmentMetadata segmentMetadata =
SegmentMetadataMockUtils.mockSegmentMetadata(TABLE_NAME, "s" + i);
- TEST_INSTANCE.getHelixResourceManager()
-
.addNewSegment(TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME),
segmentMetadata, "downloadUrl");
+ SegmentMetadata segmentMetadata =
SegmentMetadataMockUtils.mockSegmentMetadata(rawTableName, "s" + i);
+ resourceManager.addNewSegment(offlineTableName, segmentMetadata,
"downloadUrl");
segmentMetadataTable.put(segmentMetadata.getName(), segmentMetadata);
}
// There should be no segment lineage at this point.
- String segmentLineageResponse =
ControllerTest.sendGetRequest(TEST_INSTANCE.getControllerRequestURLBuilder()
- .forListAllSegmentLineages(TABLE_NAME, TableType.OFFLINE.toString()));
+ ControllerRequestURLBuilder urlBuilder =
TEST_INSTANCE.getControllerRequestURLBuilder();
+ String segmentLineageResponse =
+
ControllerTest.sendGetRequest(urlBuilder.forListAllSegmentLineages(rawTableName,
TableType.OFFLINE.name()));
assertEquals(segmentLineageResponse, "");
// Now starts to replace segments.
List<String> segmentsFrom = Arrays.asList("s0", "s1");
- List<String> segmentsTo = Arrays.asList("some_segment");
- String segmentLineageId = TEST_INSTANCE.getHelixResourceManager()
- .startReplaceSegments(TABLE_NAME_OFFLINE, segmentsFrom, segmentsTo,
false);
+ List<String> segmentsTo = Collections.singletonList("some_segment");
+ String segmentLineageId =
resourceManager.startReplaceSegments(offlineTableName, segmentsFrom,
segmentsTo, false);
// Replace more segments to add another entry to segment lineage.
segmentsFrom = Arrays.asList("s2", "s3");
- segmentsTo = Arrays.asList("another_segment");
- String nextSegmentLineageId = TEST_INSTANCE.getHelixResourceManager()
- .startReplaceSegments(TABLE_NAME_OFFLINE, segmentsFrom, segmentsTo,
false);
+ segmentsTo = Collections.singletonList("another_segment");
+ String nextSegmentLineageId =
+ resourceManager.startReplaceSegments(offlineTableName, segmentsFrom,
segmentsTo, false);
// There should now be two segment lineage entries resulting from the
operations above.
- segmentLineageResponse =
ControllerTest.sendGetRequest(TEST_INSTANCE.getControllerRequestURLBuilder()
- .forListAllSegmentLineages(TABLE_NAME, TableType.OFFLINE.toString()));
+ segmentLineageResponse =
+
ControllerTest.sendGetRequest(urlBuilder.forListAllSegmentLineages(rawTableName,
TableType.OFFLINE.toString()));
assertTrue(segmentLineageResponse.contains("\"state\":\"IN_PROGRESS\""));
assertTrue(segmentLineageResponse.contains("\"segmentsFrom\":[\"s0\",\"s1\"]"));
assertTrue(segmentLineageResponse.contains("\"segmentsTo\":[\"some_segment\"]"));
@@ -103,93 +101,77 @@ public class PinotSegmentRestletResourceTest {
assertTrue(segmentLineageResponse.indexOf(segmentLineageId) <
segmentLineageResponse.indexOf(nextSegmentLineageId));
// List segment lineage should fail for non-existing table
- assertThrows(IOException.class, () ->
ControllerTest.sendGetRequest(TEST_INSTANCE.getControllerRequestURLBuilder()
- .forListAllSegmentLineages("non-existing-table",
TableType.OFFLINE.toString())));
-
- // List segment lineage should also fail for invalid table type.
assertThrows(IOException.class, () -> ControllerTest.sendGetRequest(
-
TEST_INSTANCE.getControllerRequestURLBuilder().forListAllSegmentLineages(TABLE_NAME,
"invalid-type")));
-
- // Delete segments
-
TEST_INSTANCE.getHelixResourceManager().deleteSegment(TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME),
- segmentMetadataTable.values().iterator().next().getName());
+ urlBuilder.forListAllSegmentLineages("non-existing-table",
TableType.OFFLINE.toString())));
- // Delete offline table
- TEST_INSTANCE.getHelixResourceManager().deleteOfflineTable(TABLE_NAME);
+ // List segment lineage should also fail for invalid table type.
+ assertThrows(IOException.class,
+ () ->
ControllerTest.sendGetRequest(urlBuilder.forListAllSegmentLineages(rawTableName,
"invalid-type")));
}
@Test
public void testSegmentCrcApi()
throws Exception {
// Adding table
+ String rawTableName = "crcTestTable";
+ String offlineTableName =
TableNameBuilder.OFFLINE.tableNameWithType(rawTableName);
TableConfig tableConfig =
- new
TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).setNumReplicas(1).build();
- TEST_INSTANCE.getHelixResourceManager().addTable(tableConfig);
-
- // Wait for the table addition
- while
(!TEST_INSTANCE.getHelixResourceManager().hasOfflineTable(TABLE_NAME)) {
- Thread.sleep(100);
- }
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(rawTableName).setNumReplicas(1).build();
+ PinotHelixResourceManager resourceManager =
TEST_INSTANCE.getHelixResourceManager();
+ resourceManager.addTable(tableConfig);
// Check when there is no segment.
Map<String, SegmentMetadata> segmentMetadataTable = new HashMap<>();
- checkCrcRequest(segmentMetadataTable, 0);
+ checkCrcRequest(rawTableName, segmentMetadataTable, 0);
// Upload Segments
for (int i = 0; i < 5; i++) {
- SegmentMetadata segmentMetadata =
SegmentMetadataMockUtils.mockSegmentMetadata(TABLE_NAME);
- TEST_INSTANCE.getHelixResourceManager()
-
.addNewSegment(TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME),
segmentMetadata, "downloadUrl");
+ SegmentMetadata segmentMetadata =
SegmentMetadataMockUtils.mockSegmentMetadata(rawTableName);
+ resourceManager.addNewSegment(offlineTableName, segmentMetadata,
"downloadUrl");
segmentMetadataTable.put(segmentMetadata.getName(), segmentMetadata);
}
// Get crc info from API and check that they are correct.
- checkCrcRequest(segmentMetadataTable, 5);
+ checkCrcRequest(rawTableName, segmentMetadataTable, 5);
// validate the segment metadata
- Map.Entry<String, SegmentMetadata> entry =
- (Map.Entry<String, SegmentMetadata>)
segmentMetadataTable.entrySet().toArray()[0];
+ String sampleSegment = segmentMetadataTable.keySet().iterator().next();
String resp = ControllerTest.sendGetRequest(
-
TEST_INSTANCE.getControllerRequestURLBuilder().forSegmentMetadata(TABLE_NAME,
entry.getKey()));
+
TEST_INSTANCE.getControllerRequestURLBuilder().forSegmentMetadata(rawTableName,
sampleSegment));
Map<String, String> fetchedMetadata = JsonUtils.stringToObject(resp,
Map.class);
assertEquals(fetchedMetadata.get("segment.download.url"), "downloadUrl");
// use table name with table type
resp = ControllerTest.sendGetRequest(
-
TEST_INSTANCE.getControllerRequestURLBuilder().forSegmentMetadata(TABLE_NAME +
"_OFFLINE", entry.getKey()));
+
TEST_INSTANCE.getControllerRequestURLBuilder().forSegmentMetadata(offlineTableName,
sampleSegment));
fetchedMetadata = JsonUtils.stringToObject(resp, Map.class);
assertEquals(fetchedMetadata.get("segment.download.url"), "downloadUrl");
// Add more segments
for (int i = 0; i < 5; i++) {
- SegmentMetadata segmentMetadata =
SegmentMetadataMockUtils.mockSegmentMetadata(TABLE_NAME);
- TEST_INSTANCE.getHelixResourceManager()
-
.addNewSegment(TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME),
segmentMetadata, "downloadUrl");
+ SegmentMetadata segmentMetadata =
SegmentMetadataMockUtils.mockSegmentMetadata(rawTableName);
+ resourceManager.addNewSegment(offlineTableName, segmentMetadata,
"downloadUrl");
segmentMetadataTable.put(segmentMetadata.getName(), segmentMetadata);
}
// Get crc info from API and check that they are correct.
- checkCrcRequest(segmentMetadataTable, 10);
+ checkCrcRequest(rawTableName, segmentMetadataTable, 10);
- // Delete segments
-
TEST_INSTANCE.getHelixResourceManager().deleteSegment(TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME),
- segmentMetadataTable.values().iterator().next().getName());
+ // Delete one segment
+ resourceManager.deleteSegment(offlineTableName, sampleSegment);
// Check crc api
- checkCrcRequest(segmentMetadataTable, 9);
-
- // Delete offline table
- TEST_INSTANCE.getHelixResourceManager().deleteOfflineTable(TABLE_NAME);
+ checkCrcRequest(rawTableName, segmentMetadataTable, 9);
}
- private void checkCrcRequest(Map<String, SegmentMetadata> metadataTable, int
expectedSize)
+ private void checkCrcRequest(String tableName, Map<String, SegmentMetadata>
metadataTable, int expectedSize)
throws Exception {
String crcMapStr = ControllerTest.sendGetRequest(
-
TEST_INSTANCE.getControllerRequestURLBuilder().forListAllCrcInformationForTable(TABLE_NAME));
+
TEST_INSTANCE.getControllerRequestURLBuilder().forListAllCrcInformationForTable(tableName));
Map<String, String> crcMap = JsonUtils.stringToObject(crcMapStr,
Map.class);
for (String segmentName : crcMap.keySet()) {
SegmentMetadata metadata = metadataTable.get(segmentName);
- assertTrue(metadata != null);
+ assertNotNull(metadata);
assertEquals(crcMap.get(segmentName), metadata.getCrc());
}
assertEquals(crcMap.size(), expectedSize);
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
index bad59e60b8..41b34a4911 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
@@ -26,6 +26,7 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -76,6 +77,7 @@ import org.apache.pinot.spi.utils.CommonConstants.Server;
import org.apache.pinot.spi.utils.NetUtils;
import org.apache.pinot.spi.utils.builder.ControllerRequestURLBuilder;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.util.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -663,6 +665,11 @@ public class ControllerTest {
getControllerRequestClient().deleteTable(TableNameBuilder.REALTIME.tableNameWithType(tableName));
}
+ public void waitForEVToDisappear(String tableNameWithType) {
+ TestUtils.waitForCondition(aVoid ->
_helixResourceManager.getTableExternalView(tableNameWithType) == null, 60_000L,
+ "Failed to clean up the external view for table: " +
tableNameWithType);
+ }
+
public List<String> listSegments(String tableName)
throws IOException {
return listSegments(tableName, null, false);
@@ -984,19 +991,28 @@ public class ControllerTest {
* test functionality.
*/
public void cleanup() {
-
- // Delete all tables.
- List<String> tables = getHelixResourceManager().getAllTables();
- for (String table : tables) {
- getHelixResourceManager().deleteOfflineTable(table);
- getHelixResourceManager().deleteRealtimeTable(table);
+ // Delete all tables
+ List<String> tables = _helixResourceManager.getAllTables();
+ for (String tableNameWithType : tables) {
+ if (TableNameBuilder.isOfflineTableResource(tableNameWithType)) {
+ _helixResourceManager.deleteOfflineTable(tableNameWithType);
+ } else {
+ _helixResourceManager.deleteRealtimeTable(tableNameWithType);
+ }
}
+ // Wait for all external views to disappear
+ Set<String> tablesWithEV = new HashSet<>(tables);
+ TestUtils.waitForCondition(aVoid -> {
+ tablesWithEV.removeIf(t -> _helixResourceManager.getTableExternalView(t)
== null);
+ return tablesWithEV.isEmpty();
+ }, 60_000L, "Failed to clean up all the external views");
+
// Delete all schemas.
- List<String> schemaNames = getHelixResourceManager().getSchemaNames();
+ List<String> schemaNames = _helixResourceManager.getSchemaNames();
if (CollectionUtils.isNotEmpty(schemaNames)) {
for (String schemaName : schemaNames) {
-
getHelixResourceManager().deleteSchema(getHelixResourceManager().getSchema(schemaName));
+ getHelixResourceManager().deleteSchema(schemaName);
}
}
}
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
index 53697a7627..ebaf1746ec 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
@@ -196,7 +196,7 @@ public class TableCacheTest {
assertNull(tableCache.getColumnNameMap(RAW_TABLE_NAME));
// Remove the schema
- TEST_INSTANCE.getHelixResourceManager().deleteSchema(schema);
+ TEST_INSTANCE.getHelixResourceManager().deleteSchema(SCHEMA_NAME);
// Wait for at most 10 seconds for the callback to remove the schema from
the cache
// NOTE:
// - Verify if the callback is fully done by checking the schema change
lister because it is the last step of the
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
index a611a2f549..71f388462d 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
@@ -279,6 +279,7 @@ public class PinotHelixResourceManagerStatelessTest extends
ControllerTest {
IdealState brokerResource = HelixHelper.getBrokerIdealStates(_helixAdmin,
_clusterName);
assertTrue(brokerResource.getPartitionSet().isEmpty());
+ waitForEVToDisappear(OFFLINE_TABLE_NAME);
resetBrokerTags();
}
@@ -322,6 +323,7 @@ public class PinotHelixResourceManagerStatelessTest extends
ControllerTest {
// Delete the table
_helixResourceManager.deleteOfflineTable(RAW_TABLE_NAME);
+ waitForEVToDisappear(OFFLINE_TABLE_NAME);
// Reset the brokers
resetBrokerTags();
@@ -395,6 +397,9 @@ public class PinotHelixResourceManagerStatelessTest extends
ControllerTest {
return externalView.getStateMap(OFFLINE_TABLE_NAME) == null
&& externalView.getStateMap(REALTIME_TABLE_NAME) == null;
}, 60_000L, "Failed to get all brokers DROPPED");
+
+ waitForEVToDisappear(REALTIME_TABLE_NAME);
+ waitForEVToDisappear(OFFLINE_TABLE_NAME);
}
private void waitForTableOnlineInBrokerResourceEV(String tableNameWithType) {
@@ -988,6 +993,7 @@ public class PinotHelixResourceManagerStatelessTest extends
ControllerTest {
_helixResourceManager.deleteOfflineTable(RAW_TABLE_NAME);
segmentLineage =
SegmentLineageAccessHelper.getSegmentLineage(_propertyStore,
OFFLINE_TABLE_NAME);
assertNull(segmentLineage);
+ waitForEVToDisappear(OFFLINE_TABLE_NAME);
}
@Test
@@ -1257,6 +1263,7 @@ public class PinotHelixResourceManagerStatelessTest
extends ControllerTest {
_helixResourceManager.deleteOfflineTable(RAW_TABLE_NAME);
segmentLineage =
SegmentLineageAccessHelper.getSegmentLineage(_propertyStore,
OFFLINE_TABLE_NAME);
assertNull(segmentLineage);
+ waitForEVToDisappear(OFFLINE_TABLE_NAME);
}
private static void assertSetEquals(Collection<String> actual, String...
expected) {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentUploadIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentUploadIntegrationTest.java
index c85b494c29..5b24c97fb4 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentUploadIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentUploadIntegrationTest.java
@@ -43,6 +43,7 @@ import
org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.ingestion.batch.spec.TableSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.ControllerRequestURLBuilder;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
@@ -362,7 +363,9 @@ public class SegmentUploadIntegrationTest extends
BaseClusterIntegrationTest {
@AfterMethod
public void tearDownTest()
throws IOException {
- dropOfflineTable(getTableName());
+ String offlineTableName =
TableNameBuilder.OFFLINE.tableNameWithType(getTableName());
+ dropOfflineTable(offlineTableName);
+ waitForEVToDisappear(offlineTableName);
}
@AfterClass
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
