This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 03121cf455 fix hybrid quickstart; add @Produces to new timeBoundary
APIs (#9564)
03121cf455 is described below
commit 03121cf4553ebbd0251df028a55387f96db0b619
Author: Johan Adami <[email protected]>
AuthorDate: Tue Oct 11 16:31:22 2022 -0400
fix hybrid quickstart; add @Produces to new timeBoundary APIs (#9564)
---
.../api/resources/PinotTableRestletResource.java | 2 +
.../org/apache/pinot/tools/BootstrapTableTool.java | 81 +++++++++++++++++-----
2 files changed, 67 insertions(+), 16 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index af55fa343f..e320e3e468 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -869,6 +869,7 @@ public class PinotTableRestletResource {
@Path("tables/{tableName}/timeBoundary")
@ApiOperation(value = "Set hybrid table query time boundary based on offline
segments' metadata", notes = "Set "
+ "hybrid table query time boundary based on offline segments' metadata")
+ @Produces(MediaType.APPLICATION_JSON)
public SuccessResponse setTimeBoundary(
@ApiParam(value = "Name of the hybrid table (without type suffix)",
required = true) @PathParam("tableName")
String tableName)
@@ -907,6 +908,7 @@ public class PinotTableRestletResource {
@DELETE
@Path("tables/{tableName}/timeBoundary")
@ApiOperation(value = "Delete hybrid table query time boundary", notes =
"Delete hybrid table query time boundary")
+ @Produces(MediaType.APPLICATION_JSON)
public SuccessResponse deleteTimeBoundary(
@ApiParam(value = "Name of the hybrid table (without type suffix)",
required = true) @PathParam("tableName")
String tableName) {
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
index 36ef202758..97fcd3738f 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
+import java.io.IOException;
import java.io.Reader;
import java.net.URI;
import java.net.URL;
@@ -70,8 +71,8 @@ public class BootstrapTableTool {
_controllerHost = controllerHost;
_controllerPort = controllerPort;
_tableDir = tableDir;
- _minionClient = new MinionClient(String.format("%s://%s:%s",
controllerProtocol, controllerHost, controllerPort),
- authProvider);
+ _minionClient =
+ new MinionClient(String.format("%s://%s:%s", controllerProtocol,
controllerHost, controllerPort), authProvider);
_authProvider = authProvider;
}
@@ -89,18 +90,23 @@ public class BootstrapTableTool {
}
boolean tableCreationResult = false;
File offlineTableConfigFile = new File(tableDir,
String.format("%s_offline_table_config.json", tableName));
- if (offlineTableConfigFile.exists()) {
+ File realtimeTableConfigFile = new File(tableDir,
String.format("%s_realtime_table_config.json", tableName));
+
+ if (offlineTableConfigFile.exists() && realtimeTableConfigFile.exists()) {
+ File ingestionJobSpecFile = new File(tableDir, "ingestionJobSpec.yaml");
+ tableCreationResult =
+ bootstrapHybridTable(setupTableTmpDir, tableName, schemaFile,
offlineTableConfigFile, ingestionJobSpecFile,
+ realtimeTableConfigFile);
+ } else if (offlineTableConfigFile.exists()) {
File ingestionJobSpecFile = new File(tableDir, "ingestionJobSpec.yaml");
tableCreationResult =
bootstrapOfflineTable(setupTableTmpDir, tableName, schemaFile,
offlineTableConfigFile, ingestionJobSpecFile);
- }
- File realtimeTableConfigFile = new File(tableDir,
String.format("%s_realtime_table_config.json", tableName));
- if (realtimeTableConfigFile.exists()) {
+ } else if (realtimeTableConfigFile.exists()) {
tableCreationResult = bootstrapRealtimeTable(tableName, schemaFile,
realtimeTableConfigFile);
}
if (!tableCreationResult) {
- throw new RuntimeException(String
- .format("Unable to find config files for table - %s, at location
[%s] or [%s].", tableName,
+ throw new RuntimeException(
+ String.format("Unable to find config files for table - %s, at
location [%s] or [%s].", tableName,
offlineTableConfigFile.getAbsolutePath(),
realtimeTableConfigFile.getAbsolutePath()));
}
return true;
@@ -110,9 +116,9 @@ public class BootstrapTableTool {
throws Exception {
LOGGER.info("Adding realtime table {}", tableName);
if (!createTable(schemaFile, realtimeTableConfigFile)) {
- throw new RuntimeException(String
- .format("Unable to create realtime table - %s from schema file [%s]
and table conf file [%s].", tableName,
- schemaFile, realtimeTableConfigFile));
+ throw new RuntimeException(
+ String.format("Unable to create realtime table - %s from schema file
[%s] and table conf file [%s].",
+ tableName, schemaFile, realtimeTableConfigFile));
}
return true;
}
@@ -125,6 +131,43 @@ public class BootstrapTableTool {
.setAuthProvider(_authProvider).execute();
}
+ private boolean createTable(File schemaFile, File offlineTableConfigFile,
File realtimeTableConfigFile)
+ throws Exception {
+ return new AddTableCommand().setSchemaFile(schemaFile.getAbsolutePath())
+ .setOfflineTableConfigFile(offlineTableConfigFile.getAbsolutePath())
+ .setRealtimeTableConfigFile(realtimeTableConfigFile.getAbsolutePath())
+
.setControllerProtocol(_controllerProtocol).setControllerHost(_controllerHost)
+
.setControllerPort(String.valueOf(_controllerPort)).setExecute(true).setAuthProvider(_authProvider).execute();
+ }
+
+ private boolean bootstrapHybridTable(File setupTableTmpDir, String
tableName, File schemaFile,
+ File offlineTableConfigFile, File ingestionJobSpecFile, File
realtimeTableConfig)
+ throws Exception {
+ TableConfig tableConfig =
+ JsonUtils.inputStreamToObject(new
FileInputStream(offlineTableConfigFile), TableConfig.class);
+ if (tableConfig.getIngestionConfig() != null
+ && tableConfig.getIngestionConfig().getBatchIngestionConfig() != null)
{
+ updatedTableConfig(tableConfig, setupTableTmpDir);
+ }
+
+ LOGGER.info("Adding offline table: {}", tableName);
+ File updatedTableConfigFile =
+ new File(setupTableTmpDir, String.format("%s_%d.config", tableName,
System.currentTimeMillis()));
+ FileOutputStream outputStream = new
FileOutputStream(updatedTableConfigFile);
+ outputStream.write(JsonUtils.objectToPrettyString(tableConfig).getBytes());
+ outputStream.close();
+ // this function is separated from bootstrap offline/realtime to create
both tables at the same time in
+ // order to avoid a "table already exists error"
+ // TODO: it's unclear why this broke as it's been working, but using the
PUT API would solve this
+ boolean tableCreationResult = createTable(schemaFile,
updatedTableConfigFile, realtimeTableConfig);
+ if (!tableCreationResult) {
+ throw new RuntimeException(
+ String.format("Unable to create offline table - %s from schema file
[%s] and table conf file [%s].",
+ tableName, schemaFile, offlineTableConfigFile));
+ }
+ return setupOfflineData(setupTableTmpDir, tableConfig, tableName,
ingestionJobSpecFile);
+ }
+
private boolean bootstrapOfflineTable(File setupTableTmpDir, String
tableName, File schemaFile,
File offlineTableConfigFile, File ingestionJobSpecFile)
throws Exception {
@@ -143,14 +186,20 @@ public class BootstrapTableTool {
outputStream.close();
boolean tableCreationResult = createTable(schemaFile,
updatedTableConfigFile);
if (!tableCreationResult) {
- throw new RuntimeException(String
- .format("Unable to create offline table - %s from schema file [%s]
and table conf file [%s].", tableName,
- schemaFile, offlineTableConfigFile));
+ throw new RuntimeException(
+ String.format("Unable to create offline table - %s from schema file
[%s] and table conf file [%s].",
+ tableName, schemaFile, offlineTableConfigFile));
}
+ return setupOfflineData(setupTableTmpDir, tableConfig, tableName,
ingestionJobSpecFile);
+ }
+
+ private boolean setupOfflineData(File setupTableTmpDir, TableConfig
tableConfig, String tableName,
+ File ingestionJobSpecFile)
+ throws IOException {
if (tableConfig.getTaskConfig() != null && tableConfig.getTaskConfig()
.isTaskTypeEnabled(MinionConstants.SegmentGenerationAndPushTask.TASK_TYPE)) {
- final Map<String, String> scheduledTasks = _minionClient
-
.scheduleMinionTasks(MinionConstants.SegmentGenerationAndPushTask.TASK_TYPE,
+ final Map<String, String> scheduledTasks =
+
_minionClient.scheduleMinionTasks(MinionConstants.SegmentGenerationAndPushTask.TASK_TYPE,
TableNameBuilder.OFFLINE.tableNameWithType(tableName));
if (scheduledTasks.isEmpty()) {
LOGGER.info("No scheduled tasks.");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
