This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 11272b7eb2 Added optional force param to the table configs update API
(#10441)
11272b7eb2 is described below
commit 11272b7eb25b915df617b783f28b06fb5ef9c87d
Author: Ragesh Rajagopalan <[email protected]>
AuthorDate: Thu Mar 23 17:25:58 2023 -0700
Added optional force param to the table configs update API (#10441)
---
.../api/resources/PinotSchemaRestletResource.java | 2 +-
.../api/resources/TableConfigsRestletResource.java | 11 ++++--
.../helix/core/PinotHelixResourceManager.java | 8 ++---
.../api/TableConfigsRestletResourceTest.java | 40 ++++++++++++++++++++++
.../pinot/controller/helix/TableCacheTest.java | 2 +-
5 files changed, 54 insertions(+), 9 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
index 4104256140..66d26c1264 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
@@ -396,7 +396,7 @@ public class PinotSchemaRestletResource {
}
try {
- _pinotHelixResourceManager.updateSchema(schema, reload);
+ _pinotHelixResourceManager.updateSchema(schema, reload, false);
// Best effort notification. If controller fails at this point, no
notification is given.
LOGGER.info("Notifying metadata event for updating schema: {}",
schemaName);
_metadataEventNotifierFactory.create().notifyOnSchemaEvents(schema,
SchemaEventType.UPDATE);
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
index 8d8daff4d5..336aa3472e 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
@@ -291,6 +291,9 @@ public class TableConfigsRestletResource {
* Updated the {@link TableConfigs} by updating the schema tableName,
* then updating the offline tableConfig or creating a new one if it doesn't
already exist in the cluster,
* then updating the realtime tableConfig or creating a new one if it
doesn't already exist in the cluster.
+ *
+ * The option to skip table config validation (validationTypesToSkip) and
force update the table schema
+ * (forceTableSchemaUpdate) are provided for testing purposes and should be
used with caution.
*/
@PUT
@Path("/tableConfigs/{tableName}")
@@ -304,8 +307,10 @@ public class TableConfigsRestletResource {
@ApiParam(value = "comma separated list of validation type(s) to skip.
supported types: (ALL|TASK|UPSERT)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip,
@ApiParam(value = "Reload the table if the new schema is backward
compatible") @DefaultValue("false")
- @QueryParam("reload") boolean reload, String tableConfigsStr)
- throws Exception {
+ @QueryParam("reload") boolean reload,
+ @ApiParam(value = "Force update the table schema") @DefaultValue("false")
+ @QueryParam("forceTableSchemaUpdate") boolean forceTableSchemaUpdate,
+ String tableConfigsStr) throws Exception {
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps;
TableConfigs tableConfigs;
try {
@@ -333,7 +338,7 @@ public class TableConfigsRestletResource {
Schema schema = tableConfigs.getSchema();
try {
- _pinotHelixResourceManager.updateSchema(schema, reload);
+ _pinotHelixResourceManager.updateSchema(schema, reload,
forceTableSchemaUpdate);
LOGGER.info("Updated schema: {}", tableName);
if (offlineTableConfig != null) {
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 0cb468a04d..84ef273224 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1332,7 +1332,7 @@ public class PinotHelixResourceManager {
}
}
- public void updateSchema(Schema schema, boolean reload)
+ public void updateSchema(Schema schema, boolean reload, boolean
forceTableSchemaUpdate)
throws SchemaNotFoundException, SchemaBackwardIncompatibleException,
TableNotFoundException {
String schemaName = schema.getSchemaName();
LOGGER.info("Updating schema: {} with reload: {}", schemaName, reload);
@@ -1342,7 +1342,7 @@ public class PinotHelixResourceManager {
throw new SchemaNotFoundException(String.format("Schema: %s does not
exist", schemaName));
}
- updateSchema(schema, oldSchema, false);
+ updateSchema(schema, oldSchema, forceTableSchemaUpdate);
if (reload) {
LOGGER.info("Reloading tables with name: {}", schemaName);
@@ -1357,7 +1357,7 @@ public class PinotHelixResourceManager {
* Helper method to update the schema, or throw
SchemaBackwardIncompatibleException when the new schema is not
* backward-compatible with the existing schema.
*/
- private void updateSchema(Schema schema, Schema oldSchema, boolean force)
+ private void updateSchema(Schema schema, Schema oldSchema, boolean
forceTableSchemaUpdate)
throws SchemaBackwardIncompatibleException {
String schemaName = schema.getSchemaName();
schema.updateBooleanFieldsIfNeeded(oldSchema);
@@ -1367,7 +1367,7 @@ public class PinotHelixResourceManager {
}
boolean isBackwardCompatible = schema.isBackwardCompatibleWith(oldSchema);
if (!isBackwardCompatible) {
- if (force) {
+ if (forceTableSchemaUpdate) {
LOGGER.warn("Force updated schema: {} which is backward incompatible
with the existing schema", oldSchema);
} else {
// TODO: Add the reason of the incompatibility
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
index a506f38ac3..2ee5bb1361 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
@@ -485,6 +485,46 @@ public class TableConfigsRestletResourceTest extends
ControllerTest {
sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsDelete(tableName));
}
+ @Test
+ public void testForceUpdateTableSchemaAndConfigs()
+ throws IOException {
+ String tableName = "testUpdate1";
+ TableConfig offlineTableConfig = createOfflineTableConfig(tableName);
+ Schema schema = createDummySchema(tableName);
+ TableConfigs tableConfigs = new TableConfigs(tableName, schema,
offlineTableConfig, null);
+
+ sendPostRequest(_createTableConfigsUrl, tableConfigs.toPrettyJsonString());
+ String response =
sendGetRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsGet(tableName));
+ TableConfigs tableConfigsResponse = JsonUtils.stringToObject(response,
TableConfigs.class);
+ Assert.assertNotNull(tableConfigs.getOffline());
+
+ // Remove field from schema and try to update schema without the
'forceTableSchemaUpdate' option
+ schema.removeField("dimA");
+ tableConfigs =
+ new TableConfigs(tableName, schema, tableConfigsResponse.getOffline(),
tableConfigsResponse.getRealtime());
+
+ String tableConfigUpdateUrl =
DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsUpdate(tableName);
+ try {
+ sendPutRequest(tableConfigUpdateUrl, tableConfigs.toPrettyJsonString());
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is not backward-compatible
with the existing schema"));
+ }
+
+ // Skip validate table configs – Exception is still thrown
+ String newTableConfigUpdateUrl = tableConfigUpdateUrl +
"?validationTypesToSkip=ALL";
+ try {
+ sendPutRequest(newTableConfigUpdateUrl,
tableConfigs.toPrettyJsonString());
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is not backward-compatible
with the existing schema"));
+ }
+
+ // Skip table config validation as well as force update the table schema –
no exceptions are thrown
+ newTableConfigUpdateUrl = tableConfigUpdateUrl +
"?validationTypesToSkip=ALL&forceTableSchemaUpdate=true";
+ response = sendPutRequest(newTableConfigUpdateUrl,
tableConfigs.toPrettyJsonString());
+ Assert.assertTrue(response.contains("TableConfigs updated for
testUpdate1"));
+
sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsDelete(tableName));
+ }
+
@Test
public void testDeleteConfig()
throws Exception {
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
index 9f2024dd99..53697a7627 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java
@@ -128,7 +128,7 @@ public class TableCacheTest {
// Update the schema
schema.addField(new DimensionFieldSpec("newColumn", DataType.LONG, true));
- TEST_INSTANCE.getHelixResourceManager().updateSchema(schema, false);
+ TEST_INSTANCE.getHelixResourceManager().updateSchema(schema, false, false);
// Wait for at most 10 seconds for the callback to update the schema in
the cache
// NOTE:
// - Schema should never be null during the transitioning
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
