This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch adding_cluster_config_ops in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 4a739b120f7c19fa441aa74adf88bbd47e2ff47e Author: Xiang Fu <[email protected]> AuthorDate: Sun Feb 16 02:23:18 2020 -0800 Adding Pinot Cluster Config API in controller and corresponding PinotAdmin Commands --- .../pinot/controller/api/resources/Constants.java | 1 + .../api/resources/PinotClusterConfigs.java | 100 +++++++++++++++++++ .../pinot/tools/admin/PinotAdministrator.java | 4 + .../admin/command/AbstractBaseAdminCommand.java | 1 + .../admin/command/GetClusterConfigsCommand.java | 108 +++++++++++++++++++++ .../admin/command/UpdateClusterConfigCommand.java | 107 ++++++++++++++++++++ 6 files changed, 321 insertions(+) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java index 0278105..2aea519 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; public class Constants { private static final Logger LOGGER = LoggerFactory.getLogger(Constants.class); + public static final String CLUSTER_TAG = "Cluster"; public static final String TABLE_TAG = "Table"; public static final String VERSION_TAG = "Version"; public static final String HEALTH_TAG = "Health"; diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java new file mode 100644 index 0000000..b553f0a --- /dev/null +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.controller.api.resources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.helix.HelixAdmin; +import org.apache.helix.model.HelixConfigScope; +import org.apache.helix.model.builder.HelixConfigScopeBuilder; +import org.apache.pinot.controller.helix.core.PinotHelixResourceManager; +import org.apache.pinot.spi.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +@Api(tags = Constants.CLUSTER_TAG) +@Path("/") +public class PinotClusterConfigs { + private static final Logger LOGGER = LoggerFactory.getLogger(PinotClusterConfigs.class); + + @Inject + PinotHelixResourceManager pinotHelixResourceManager; + + @GET + @Path("/cluster/configs") + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "List cluster configurations", notes = "List cluster level configurations") + @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error")}) + public String getTableInstances() { + HelixAdmin helixAdmin = pinotHelixResourceManager.getHelixAdmin(); + HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER) + .forCluster(pinotHelixResourceManager.getHelixClusterName()).build(); + List<String> configKeys = helixAdmin.getConfigKeys(configScope); + ObjectNode ret = JsonUtils.newObjectNode(); + Map<String, String> configs = helixAdmin.getConfig(configScope, configKeys); + for (String key : configs.keySet()) { + ret.put(key, configs.get(key)); + } + return ret.toString(); + } + + @POST + @Path("/cluster/configs") + @ApiOperation(value = "Update cluster configuration") + @Produces(MediaType.APPLICATION_JSON) + @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Server error updating configuration")}) + public SuccessResponse updateIndexingConfig(String body) { + try { + JsonNode jsonNode = JsonUtils.stringToJsonNode(body); + HelixAdmin admin = pinotHelixResourceManager.getHelixAdmin(); + HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER) + .forCluster(pinotHelixResourceManager.getHelixClusterName()).build(); + Iterator<String> fieldNamesIterator = jsonNode.fieldNames(); + while (fieldNamesIterator.hasNext()) { + String key = fieldNamesIterator.next(); + String value = jsonNode.get(key).textValue(); + admin.setConfig(configScope, Collections.singletonMap(key, value)); + } + return new SuccessResponse("Updated cluster config."); + } catch (IOException e) { + String errStr = "Error converting request to cluster config."; + throw new ControllerApplicationException(LOGGER, errStr, Response.Status.BAD_REQUEST, e); + } catch (Exception e) { + String errStr = "Failed to update cluster config."; + throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); + } + } +} diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java index fc712a9..3ec2d7d 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java @@ -32,6 +32,7 @@ import org.apache.pinot.tools.admin.command.ChangeTableState; import org.apache.pinot.tools.admin.command.CreateSegmentCommand; import org.apache.pinot.tools.admin.command.DeleteClusterCommand; import org.apache.pinot.tools.admin.command.GenerateDataCommand; +import org.apache.pinot.tools.admin.command.GetClusterConfigsCommand; import org.apache.pinot.tools.admin.command.LaunchDataIngestionJobCommand; import org.apache.pinot.tools.admin.command.MoveReplicaGroup; import org.apache.pinot.tools.admin.command.OfflineSegmentIntervalCheckerCommand; @@ -46,6 +47,7 @@ import org.apache.pinot.tools.admin.command.StartServerCommand; import org.apache.pinot.tools.admin.command.StartZookeeperCommand; import org.apache.pinot.tools.admin.command.StopProcessCommand; import org.apache.pinot.tools.admin.command.StreamAvroIntoKafkaCommand; +import org.apache.pinot.tools.admin.command.UpdateClusterConfigCommand; import org.apache.pinot.tools.admin.command.UploadSegmentCommand; import org.apache.pinot.tools.admin.command.ValidateConfigCommand; import org.apache.pinot.tools.admin.command.VerifyClusterStateCommand; @@ -87,6 +89,8 @@ public class PinotAdministrator { //@formatter:off @Argument(handler = SubCommandHandler.class, metaVar = "<subCommand>") @SubCommands({ + @SubCommand(name = "GetClusterConfigs", impl = GetClusterConfigsCommand.class), + @SubCommand(name = "UpdateClusterConfig", impl = UpdateClusterConfigCommand.class), @SubCommand(name = "GenerateData", impl = GenerateDataCommand.class), @SubCommand(name = "LaunchDataIngestionJob", impl = LaunchDataIngestionJobCommand.class), @SubCommand(name = "CreateSegment", impl = CreateSegmentCommand.class), diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AbstractBaseAdminCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AbstractBaseAdminCommand.java index fd56bfa..c2f84ec 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AbstractBaseAdminCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AbstractBaseAdminCommand.java @@ -29,6 +29,7 @@ import java.net.URL; import java.net.URLConnection; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.io.IOUtils; import org.apache.pinot.tools.AbstractBaseCommand; diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GetClusterConfigsCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GetClusterConfigsCommand.java new file mode 100644 index 0000000..c52078a --- /dev/null +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GetClusterConfigsCommand.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.tools.admin.command; + +import com.fasterxml.jackson.databind.JsonNode; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import org.apache.commons.io.IOUtils; +import org.apache.pinot.common.utils.NetUtil; +import org.apache.pinot.spi.utils.JsonUtils; +import org.apache.pinot.tools.Command; +import org.kohsuke.args4j.Option; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class GetClusterConfigsCommand extends AbstractBaseAdminCommand implements Command { + private static final Logger LOGGER = LoggerFactory.getLogger(GetClusterConfigsCommand.class.getName()); + + @Option(name = "-controllerHost", required = false, metaVar = "<String>", usage = "host name for controller.") + private String _controllerHost; + + @Option(name = "-controllerPort", required = false, metaVar = "<int>", usage = "http port for controller.") + private String _controllerPort = DEFAULT_CONTROLLER_PORT; + + @Option(name = "-help", required = false, help = true, aliases = {"-h", "--h", "--help"}, usage = "Print this message.") + private boolean _help = false; + + @Override + public boolean getHelp() { + return _help; + } + + @Override + public String getName() { + return "GetClusterConfigsCommand"; + } + + @Override + public String toString() { + return ("GetClusterConfigsCommand -controllerHost " + _controllerHost + " -controllerPort " + _controllerPort); + } + + @Override + public void cleanup() { + + } + + @Override + public String description() { + return "Get Pinot Cluster Configs. Sample usage: `pinot-admin.sh GetClusterConfigs`"; + } + + public GetClusterConfigsCommand setControllerHost(String host) { + _controllerHost = host; + return this; + } + + public GetClusterConfigsCommand setControllerPort(String port) { + _controllerPort = port; + return this; + } + + public String run() + throws Exception { + if (_controllerHost == null) { + _controllerHost = NetUtil.getHostAddress(); + } + LOGGER.info("Executing command: " + toString()); + String response = IOUtils + .toString(new URI("http://"; + _controllerHost + ":" + _controllerPort + "/cluster/configs"), + StandardCharsets.UTF_8); + JsonNode jsonNode = JsonUtils.stringToJsonNode(response); + Iterator<String> fieldNamesIterator = jsonNode.fieldNames(); + String results = ""; + while (fieldNamesIterator.hasNext()) { + String key = fieldNamesIterator.next(); + String value = jsonNode.get(key).textValue(); + results += String.format("%s=%s\n", key, value); + } + return results; + } + + @Override + public boolean execute() + throws Exception { + String result = run(); + LOGGER.info(result); + return true; + } +} diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UpdateClusterConfigCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UpdateClusterConfigCommand.java new file mode 100644 index 0000000..08d0dcd --- /dev/null +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UpdateClusterConfigCommand.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.tools.admin.command; + +import java.util.Collections; +import org.apache.pinot.common.utils.NetUtil; +import org.apache.pinot.spi.utils.JsonUtils; +import org.apache.pinot.tools.Command; +import org.kohsuke.args4j.Option; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class UpdateClusterConfigCommand extends AbstractBaseAdminCommand implements Command { + private static final Logger LOGGER = LoggerFactory.getLogger(UpdateClusterConfigCommand.class.getName()); + + @Option(name = "-controllerHost", required = false, metaVar = "<String>", usage = "host name for controller.") + private String _controllerHost; + + @Option(name = "-controllerPort", required = false, metaVar = "<int>", usage = "http port for controller.") + private String _controllerPort = DEFAULT_CONTROLLER_PORT; + + @Option(name = "-config", required = true, metaVar = "<string>", usage = "Cluster config to update.") + private String _config; + + @Option(name = "-help", required = false, help = true, aliases = {"-h", "--h", "--help"}, usage = "Print this message.") + private boolean _help = false; + + @Override + public boolean getHelp() { + return _help; + } + + @Override + public String getName() { + return "UpdateClusterConfig"; + } + + @Override + public String toString() { + return ("UpdateClusterConfig -controllerHost " + _controllerHost + " -controllerPort " + _controllerPort + + " -config " + _config); + } + + @Override + public void cleanup() { + + } + + @Override + public String description() { + return "Update Pinot Cluster Config. Sample usage: `pinot-admin.sh UpdateClusterConfig -config pinot.broker.enable.query.limit.override=true`"; + } + + public UpdateClusterConfigCommand setControllerHost(String host) { + _controllerHost = host; + return this; + } + + public UpdateClusterConfigCommand setControllerPort(String port) { + _controllerPort = port; + return this; + } + + public UpdateClusterConfigCommand setConfig(String config) { + _config = config; + return this; + } + + public String run() + throws Exception { + if (_controllerHost == null) { + _controllerHost = NetUtil.getHostAddress(); + } + LOGGER.info("Executing command: " + toString()); + String[] splits = _config.split("="); + if (splits.length != 2) { + throw new UnsupportedOperationException("Bad config: " + _config + ". Please follow the pattern of [Config Key]=[Config Value]"); + } + String request = JsonUtils.objectToString(Collections.singletonMap(splits[0], splits[1])); + return sendPostRequest("http://"; + _controllerHost + ":" + _controllerPort + "/cluster/configs", request); + } + + @Override + public boolean execute() + throws Exception { + String result = run(); + LOGGER.info("Result: " + result); + return true; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
