This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7299b56 Add admin endpoint for Pinot Minon. (#6822)
7299b56 is described below
commit 7299b56ef3ca30a1730a24382a2701b8b2fc5c65
Author: Mayank Shrivastava <[email protected]>
AuthorDate: Tue Apr 20 18:05:15 2021 -0700
Add admin endpoint for Pinot Minon. (#6822)
The Pinot Minion component, unlike other components, was missing an admin
endpoint.
In this PR, we added an admin endpoint for Pinot Minion.
- The endpoint uses Jersey http server (same as other components).
- Setup swagger for Minion.
- Also, added the endpoint `/appconfigs` similiar to other components.
(See: https://github.com/apache/incubator-pinot/pull/6817).
---
.../apache/pinot/core/util/ListenerConfigUtil.java | 21 +++++
pinot-minion/pom.xml | 4 +
.../pinot/minion/MinionAdminApiApplication.java | 103 +++++++++++++++++++++
.../org/apache/pinot/minion/MinionStarter.java | 24 +++++
.../api/resources/PinotMinionAppConfigs.java | 52 +++++++++++
.../apache/pinot/spi/utils/CommonConstants.java | 3 +
6 files changed, 207 insertions(+)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
index 01936c2..9282ef5 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
@@ -143,6 +143,27 @@ public final class ListenerConfigUtil {
return listeners;
}
+ public static List<ListenerConfig>
buildMinionAdminConfigs(PinotConfiguration minionConf) {
+ List<ListenerConfig> listeners = new ArrayList<>();
+
+ String adminApiPortString =
minionConf.getProperty(CommonConstants.Minion.CONFIG_OF_ADMIN_API_PORT);
+ if (adminApiPortString != null) {
+ listeners.add(new ListenerConfig(CommonConstants.HTTP_PROTOCOL,
DEFAULT_HOST, Integer.parseInt(adminApiPortString),
+ CommonConstants.HTTP_PROTOCOL, new TlsConfig()));
+ }
+
+ TlsConfig tlsDefaults = TlsUtils.extractTlsConfig(minionConf,
CommonConstants.Minion.MINION_TLS_PREFIX);
+ listeners.addAll(buildListenerConfigs(minionConf, "pinot.minion.adminapi",
tlsDefaults));
+
+ // support legacy behavior < 0.7.0
+ if (listeners.isEmpty()) {
+ listeners.add(new ListenerConfig(CommonConstants.HTTP_PROTOCOL,
DEFAULT_HOST,
+ CommonConstants.Minion.DEFAULT_ADMIN_API_PORT,
CommonConstants.HTTP_PROTOCOL, new TlsConfig()));
+ }
+
+ return listeners;
+ }
+
private static ListenerConfig buildListenerConfig(PinotConfiguration config,
String namespace, String protocol,
TlsConfig tlsConfig) {
String protocolNamespace = namespace + DOT_ACCESS_PROTOCOLS + "." +
protocol;
diff --git a/pinot-minion/pom.xml b/pinot-minion/pom.xml
index 949da8d..ffe9f7f 100644
--- a/pinot-minion/pom.xml
+++ b/pinot-minion/pom.xml
@@ -97,5 +97,9 @@
<artifactId>pinot-yammer</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-jersey2-jaxrs</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
new file mode 100644
index 0000000..12b5058
--- /dev/null
+++
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.minion;
+
+import io.swagger.jaxrs.config.BeanConfig;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.List;
+import org.apache.pinot.core.transport.ListenerConfig;
+import org.apache.pinot.core.util.ListenerConfigUtil;
+import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.glassfish.grizzly.http.server.CLStaticHttpHandler;
+import org.glassfish.grizzly.http.server.HttpHandler;
+import org.glassfish.grizzly.http.server.HttpServer;
+import org.glassfish.jersey.internal.inject.AbstractBinder;
+import org.glassfish.jersey.server.ResourceConfig;
+
+
+/**
+ * Admin APP for Pinot Minion.
+ * <ul>
+ * <li>Starts a http server.</li>
+ * <li>Sets up swagger.</li>
+ * </ul>
+ */
+public class MinionAdminApiApplication extends ResourceConfig {
+ private static final String RESOURCE_PACKAGE =
"org.apache.pinot.minion.api.resources";
+ public static final String PINOT_CONFIGURATION = "pinotConfiguration";
+
+ private HttpServer _httpServer;
+
+ public MinionAdminApiApplication(PinotConfiguration minionConf) {
+ packages(RESOURCE_PACKAGE);
+ property(PINOT_CONFIGURATION, minionConf);
+
+ register(new AbstractBinder() {
+ @Override
+ protected void configure() {
+ // TODO: Add bindings as needed in future.
+ }
+ });
+
+ registerClasses(io.swagger.jaxrs.listing.ApiListingResource.class);
+ registerClasses(io.swagger.jaxrs.listing.SwaggerSerializers.class);
+ }
+
+ public void start(List<ListenerConfig> listenerConfigs) {
+ _httpServer = ListenerConfigUtil.buildHttpServer(this, listenerConfigs);
+
+ try {
+ _httpServer.start();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to start http server", e);
+ }
+
+ setupSwagger();
+ }
+
+ private void setupSwagger() {
+ BeanConfig beanConfig = new BeanConfig();
+ beanConfig.setTitle("Pinot Minion API");
+ beanConfig.setDescription("APIs for accessing Pinot Minion information");
+ beanConfig.setContact("https://github.com/apache/incubator-pinot";);
+ beanConfig.setVersion("1.0");
+ beanConfig.setSchemes(new String[]{CommonConstants.HTTP_PROTOCOL,
CommonConstants.HTTPS_PROTOCOL});
+ beanConfig.setBasePath("/");
+ beanConfig.setResourcePackage(RESOURCE_PACKAGE);
+ beanConfig.setScan(true);
+
+ HttpHandler httpHandler = new
CLStaticHttpHandler(MinionAdminApiApplication.class.getClassLoader(), "/api/");
+ // map both /api and /help to swagger docs. /api because it looks nice.
/help for backward compatibility
+ _httpServer.getServerConfiguration().addHttpHandler(httpHandler, "/api/",
"/help/");
+
+ URL swaggerDistLocation =
+
MinionAdminApiApplication.class.getClassLoader().getResource("META-INF/resources/webjars/swagger-ui/3.18.2/");
+ CLStaticHttpHandler swaggerDist = new CLStaticHttpHandler(new
URLClassLoader(new URL[]{swaggerDistLocation}));
+ _httpServer.getServerConfiguration().addHttpHandler(swaggerDist,
"/swaggerui-dist/");
+ }
+
+ public void stop() {
+ if (_httpServer != null) {
+ _httpServer.shutdownNow();
+ }
+ }
+}
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
index 0615d28..6cd3346 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
@@ -20,8 +20,10 @@ package org.apache.pinot.minion;
import java.io.File;
import java.io.IOException;
+import java.util.List;
import javax.net.ssl.SSLContext;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixManager;
import org.apache.helix.InstanceType;
@@ -34,6 +36,10 @@ import org.apache.pinot.common.metrics.PinotMetricUtils;
import org.apache.pinot.common.utils.ClientSSLContextGenerator;
import org.apache.pinot.common.utils.ServiceStatus;
import org.apache.pinot.common.utils.fetcher.SegmentFetcherFactory;
+import org.apache.pinot.core.transport.ListenerConfig;
+import org.apache.pinot.core.transport.TlsConfig;
+import org.apache.pinot.core.util.ListenerConfigUtil;
+import org.apache.pinot.core.util.TlsUtils;
import org.apache.pinot.minion.event.EventObserverFactoryRegistry;
import org.apache.pinot.minion.event.MinionEventObserverFactory;
import org.apache.pinot.minion.executor.MinionTaskZkMetadataManager;
@@ -68,6 +74,8 @@ public class MinionStarter implements ServiceStartable {
private final HelixManager _helixManager;
private final TaskExecutorFactoryRegistry _taskExecutorFactoryRegistry;
private final EventObserverFactoryRegistry _eventObserverFactoryRegistry;
+ private MinionAdminApiApplication _minionAdminApplication;
+ private final List<ListenerConfig> _listenerConfigs;
public MinionStarter(String helixClusterName, String zkAddress,
PinotConfiguration config)
throws Exception {
@@ -78,6 +86,7 @@ public class MinionStarter implements ServiceStartable {
int port = _config.getProperty(CommonConstants.Helix.KEY_OF_MINION_PORT,
CommonConstants.Minion.DEFAULT_HELIX_PORT);
_instanceId =
_config.getProperty(CommonConstants.Helix.Instance.INSTANCE_ID_KEY,
CommonConstants.Helix.PREFIX_OF_MINION_INSTANCE + host + "_" + port);
+ _listenerConfigs = ListenerConfigUtil.buildMinionAdminConfigs(_config);
setupHelixSystemProperties();
_helixManager = new ZKHelixManager(helixClusterName, _instanceId,
InstanceType.PARTICIPANT, zkAddress);
MinionTaskZkMetadataManager minionTaskZkMetadataManager = new
MinionTaskZkMetadataManager(_helixManager);
@@ -160,6 +169,14 @@ public class MinionStarter implements ServiceStartable {
minionMetrics.initializeGlobalMeters();
minionContext.setMinionMetrics(minionMetrics);
+ // Install default SSL context if necessary (even if not force-enabled
everywhere)
+ TlsConfig tlsDefaults = TlsUtils.extractTlsConfig(_config,
CommonConstants.Minion.MINION_TLS_PREFIX);
+ if (StringUtils.isNotBlank(tlsDefaults.getKeyStorePath()) || StringUtils
+ .isNotBlank(tlsDefaults.getTrustStorePath())) {
+ LOGGER.info("Installing default SSL context for any client requests");
+ TlsUtils.installDefaultSSLSocketFactory(tlsDefaults);
+ }
+
// initialize authentication
minionContext.setTaskAuthToken(_config.getProperty(CommonConstants.Minion.CONFIG_OF_TASK_AUTH_TOKEN));
@@ -195,6 +212,10 @@ public class MinionStarter implements ServiceStartable {
addInstanceTagIfNeeded();
minionContext.setHelixPropertyStore(_helixManager.getHelixPropertyStore());
+ LOGGER.info("Starting minion admin application on: {}",
ListenerConfigUtil.toString(_listenerConfigs));
+ _minionAdminApplication = new MinionAdminApiApplication(_config);
+ _minionAdminApplication.start(_listenerConfigs);
+
// Initialize health check callback
LOGGER.info("Initializing health check callback");
ServiceStatus.setServiceStatusCallback(_instanceId, new
ServiceStatus.ServiceStatusCallback() {
@@ -225,6 +246,9 @@ public class MinionStarter implements ServiceStartable {
} catch (IOException e) {
LOGGER.warn("Caught exception closing PinotFS classes", e);
}
+ LOGGER.info("Shutting down admin application");
+ _minionAdminApplication.stop();
+
LOGGER.info("Stopping Pinot minion: " + _instanceId);
_helixManager.disconnect();
LOGGER.info("Deregistering service status handler");
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionAppConfigs.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionAppConfigs.java
new file mode 100644
index 0000000..6d4b4ec
--- /dev/null
+++
b/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionAppConfigs.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.minion.api.resources;
+
+import io.swagger.annotations.Api;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import org.apache.pinot.common.utils.PinotAppConfigs;
+import org.apache.pinot.minion.MinionAdminApiApplication;
+import org.apache.pinot.spi.env.PinotConfiguration;
+
+
+/**
+ * Resource for getting the app configs {@link PinotAppConfigs} for
+ * Pinot Minion instance.
+ */
+@Api(tags = "AppConfig")
+@Path("/")
+public class PinotMinionAppConfigs {
+ @Context
+ private Application _application;
+
+ @GET
+ @Path("/appconfigs")
+ @Produces(MediaType.APPLICATION_JSON)
+ public String getAppConfigs() {
+ PinotConfiguration pinotConfiguration =
+ (PinotConfiguration)
_application.getProperties().get(MinionAdminApiApplication.PINOT_CONFIGURATION);
+ PinotAppConfigs pinotAppConfigs = new PinotAppConfigs(pinotConfiguration);
+ return pinotAppConfigs.toJSONString();
+ }
+}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 3e42d75..534a3a0 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -386,6 +386,9 @@ public class CommonConstants {
* E.g. null (auth disabled), "Basic abcdef..." (basic auth), "Bearer
123def..." (oauth2)
*/
public static final String CONFIG_OF_TASK_AUTH_TOKEN = "task.auth.token";
+ public static final String CONFIG_OF_ADMIN_API_PORT =
"pinot.minion.adminapi.port";
+ public static final String MINION_TLS_PREFIX = "pinot.minion.tls";
+ public static final int DEFAULT_ADMIN_API_PORT = 6500;
}
public static class Segment {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
