Repository: incubator-gobblin Updated Branches: refs/heads/master a889431e0 -> 15ac4679b
Making gobblin-admin a module Making gobblin-admin a module Revert gobblin-admin move Addressing review comments Closes #1540 from chavdar/admin_deps Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/15ac4679 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/15ac4679 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/15ac4679 Branch: refs/heads/master Commit: 15ac4679b3d8c2048ab5eca68f0cc1bdfe02097c Parents: a889431 Author: Chavdar Botev <cbotev@cbotev-ld.(none)> Authored: Sat Jul 29 00:57:28 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Sat Jul 29 00:57:28 2017 -0700 ---------------------------------------------------------------------- .../admin/DefaultAdminWebServerFactory.java | 39 +++++++++++++++++ .../configuration/ConfigurationKeys.java | 7 +++- .../runtime/api/AdminWebServerFactory.java | 44 ++++++++++++++++++++ gobblin-aws/build.gradle | 1 - gobblin-cluster/build.gradle | 1 - gobblin-distribution/build.gradle | 1 - .../gobblin-flavor-custom.gradle | 3 ++ gobblin-distribution/gobblin-flavor-full.gradle | 1 + gobblin-runtime/build.gradle | 1 - .../runtime/app/ServiceBasedAppLauncher.java | 25 +++++++++-- gobblin-yarn/build.gradle | 1 - .../gobblin/yarn/GobblinYarnAppLauncher.java | 9 ++-- 12 files changed, 119 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java b/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java new file mode 100644 index 0000000..87c2112 --- /dev/null +++ b/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package gobblin.admin; + +import java.net.URI; +import java.util.Properties; + +import com.google.common.util.concurrent.Service; + +import gobblin.annotation.Alias; +import gobblin.runtime.api.AdminWebServerFactory; + +/** + * A factory for the default implementation of the admin web server + */ +@Alias(value="default") +public class DefaultAdminWebServerFactory implements AdminWebServerFactory { + + /** {@inheritDoc} */ + @Override + public Service createInstance(Properties config, URI executionInfoServerURI) { + return new AdminWebServer(config, executionInfoServerURI); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java index c20a7d3..e119d43 100644 --- a/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java @@ -676,10 +676,13 @@ public class ConfigurationKeys { public static final String DEFAULT_REST_SERVER_PORT = "8080"; public static final String REST_SERVER_ADVERTISED_URI_KEY = "rest.server.advertised.uri"; - /** + /* * Admin server configuration properties. */ public static final String ADMIN_SERVER_ENABLED_KEY = "admin.server.enabled"; + /** The name of the class with the admin interface. The class must implement the + * AdminWebServerFactory interface .*/ + public static final String ADMIN_SERVER_FACTORY_CLASS_KEY = "admin.server.factory.type"; public static final String ADMIN_SERVER_HOST_KEY = "admin.server.host"; public static final String DEFAULT_ADMIN_SERVER_HOST = "localhost"; public static final String ADMIN_SERVER_PORT_KEY = "admin.server.port"; @@ -689,6 +692,8 @@ public class ConfigurationKeys { public static final String ADMIN_SERVER_REFRESH_INTERVAL_KEY = "admin.server.refresh_interval"; public static final long DEFAULT_ADMIN_SERVER_REFRESH_INTERVAL = 30000; + public static final String DEFAULT_ADMIN_SERVER_FACTORY_CLASS = "gobblin.admin.DefaultAdminWebServerFactory"; + /** * Kafka job configurations. */ http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java b/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java new file mode 100644 index 0000000..f9f998d --- /dev/null +++ b/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package gobblin.runtime.api; + +import java.net.URI; +import java.util.Properties; + +import com.google.common.util.concurrent.Service; + +import gobblin.annotation.Alpha; + +/** + * A factory interface for AdminWebServer. + * + * NOTE: This interface is provided for backwards compatibility and may change in the future. + * @author cbotev + * + */ +@Alpha +public interface AdminWebServerFactory { + + /** + * Creates a new AdminWebServer instance + * @param config the server config + * @param executionInfoServerURI the URI to the job execution server + * @return the instance + */ + Service createInstance(Properties config, URI executionInfoServerURI); + +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-aws/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-aws/build.gradle b/gobblin-aws/build.gradle index a76f449..37d4389 100644 --- a/gobblin-aws/build.gradle +++ b/gobblin-aws/build.gradle @@ -18,7 +18,6 @@ apply plugin: 'java' dependencies { - compile project(":gobblin-admin") compile project(":gobblin-api") compile project(":gobblin-cluster") compile project(":gobblin-core") http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-cluster/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-cluster/build.gradle b/gobblin-cluster/build.gradle index b96ee1f..da892aa 100644 --- a/gobblin-cluster/build.gradle +++ b/gobblin-cluster/build.gradle @@ -18,7 +18,6 @@ apply plugin: 'java' dependencies { - compile project(":gobblin-admin") compile project(":gobblin-api") compile project(":gobblin-core") compile project(":gobblin-metrics-libs:gobblin-metrics") http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-distribution/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-distribution/build.gradle b/gobblin-distribution/build.gradle index 760edeb..fff1298 100644 --- a/gobblin-distribution/build.gradle +++ b/gobblin-distribution/build.gradle @@ -29,7 +29,6 @@ configurations { } dependencies { - compile project(':gobblin-admin') compile project(':gobblin-api') compile project(':gobblin-core') compile project(':gobblin-compaction') http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-distribution/gobblin-flavor-custom.gradle ---------------------------------------------------------------------- diff --git a/gobblin-distribution/gobblin-flavor-custom.gradle b/gobblin-distribution/gobblin-flavor-custom.gradle index 4913c4f..1f935bf 100644 --- a/gobblin-distribution/gobblin-flavor-custom.gradle +++ b/gobblin-distribution/gobblin-flavor-custom.gradle @@ -19,6 +19,9 @@ dependencies { // Example jobs // compile project(':gobblin-example') + // Admin Web Server + // compile project(':gobblin-admin') + // Gobblin Azkaban integration // compile project(':gobblin-modules:gobblin-azkaban') http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-distribution/gobblin-flavor-full.gradle ---------------------------------------------------------------------- diff --git a/gobblin-distribution/gobblin-flavor-full.gradle b/gobblin-distribution/gobblin-flavor-full.gradle index 71b5d33..22ab209 100644 --- a/gobblin-distribution/gobblin-flavor-full.gradle +++ b/gobblin-distribution/gobblin-flavor-full.gradle @@ -17,6 +17,7 @@ dependencies { compile project(':gobblin-example') + compile project(':gobblin-admin') compile project(':gobblin-modules:gobblin-azkaban') compile project(':gobblin-modules:gobblin-compliance') compile project(':gobblin-modules:gobblin-couchbase') http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-runtime/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-runtime/build.gradle b/gobblin-runtime/build.gradle index 85b367a..ec99635 100644 --- a/gobblin-runtime/build.gradle +++ b/gobblin-runtime/build.gradle @@ -36,7 +36,6 @@ compileJava { } dependencies { - compile project(":gobblin-admin") compile project(":gobblin-api") compile project(":gobblin-core") compile project(":gobblin-hive-registration") http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java b/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java index 58010c5..6a07cf2 100644 --- a/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java +++ b/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java @@ -20,6 +20,7 @@ package gobblin.runtime.app; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -34,14 +35,15 @@ import com.google.common.util.concurrent.Service; import com.google.common.util.concurrent.ServiceManager; import gobblin.annotation.Alpha; -import gobblin.admin.AdminWebServer; import gobblin.configuration.ConfigurationKeys; import gobblin.configuration.State; import gobblin.metrics.GobblinMetrics; import gobblin.rest.JobExecutionInfoServer; +import gobblin.runtime.api.AdminWebServerFactory; import gobblin.runtime.services.JMXReportingService; import gobblin.runtime.services.MetricsReportingService; import gobblin.util.ApplicationLauncherUtils; +import gobblin.util.ClassAliasResolver; /** @@ -52,7 +54,7 @@ import gobblin.util.ApplicationLauncherUtils; * <ul> * <li>{@link MetricsReportingService} is optional and controlled by {@link ConfigurationKeys#METRICS_ENABLED_KEY}</li> * <li>{@link JobExecutionInfoServer} is optional and controlled by {@link ConfigurationKeys#JOB_EXECINFO_SERVER_ENABLED_KEY}</li> - * <li>{@link AdminWebServer} is optional and controlled by {@link ConfigurationKeys#ADMIN_SERVER_ENABLED_KEY}</li> + * <li>AdminWebServer is optional and controlled by {@link ConfigurationKeys#ADMIN_SERVER_ENABLED_KEY}</li> * <li>{@link JMXReportingService} is mandatory</li> * </ul> * @@ -224,13 +226,30 @@ public class ServiceBasedAppLauncher implements ApplicationLauncher { if (adminUiServerEnabled) { LOG.info("Will launch the admin UI server"); - addService(new AdminWebServer(properties, executionInfoServer.getAdvertisedServerUri())); + addService(createAdminServer(properties, executionInfoServer.getAdvertisedServerUri())); } } else if (adminUiServerEnabled) { LOG.warn("Not launching the admin UI because the job execution info server is not enabled"); } } + public static Service createAdminServer(Properties properties, + URI executionInfoServerURI) { + String factoryClassName = properties.getProperty(ConfigurationKeys.ADMIN_SERVER_FACTORY_CLASS_KEY, + ConfigurationKeys.DEFAULT_ADMIN_SERVER_FACTORY_CLASS); + ClassAliasResolver<AdminWebServerFactory> classResolver = + new ClassAliasResolver<>(AdminWebServerFactory.class); + try + { + AdminWebServerFactory factoryInstance = classResolver.resolveClass(factoryClassName).newInstance(); + return factoryInstance.createInstance(properties, executionInfoServerURI); + } + catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + throw new RuntimeException("Unable to instantiate the AdminWebServer factory. " + + "Have you included the module in the gobblin distribution? :" + e, e); + } + } + private void addMetricsService(Properties properties) { if (GobblinMetrics.isEnabled(properties)) { addService(new MetricsReportingService(properties, this.appId)); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-yarn/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-yarn/build.gradle b/gobblin-yarn/build.gradle index 1ab9558..02959bf 100644 --- a/gobblin-yarn/build.gradle +++ b/gobblin-yarn/build.gradle @@ -22,7 +22,6 @@ plugins { apply plugin: 'java' dependencies { - compile project(":gobblin-admin") compile project(":gobblin-api") compile project(":gobblin-cluster") compile project(":gobblin-core") http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java index 6603813..a5f119d 100644 --- a/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java +++ b/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java @@ -61,7 +61,6 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; - import org.apache.helix.Criteria; import org.apache.helix.HelixManager; import org.apache.helix.HelixManagerFactory; @@ -84,17 +83,16 @@ import com.google.common.io.Closer; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.Service; import com.google.common.util.concurrent.ServiceManager; - import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; -import gobblin.admin.AdminWebServer; import gobblin.cluster.GobblinClusterConfigurationKeys; import gobblin.cluster.GobblinClusterUtils; import gobblin.cluster.GobblinHelixConstants; import gobblin.cluster.HelixUtils; import gobblin.configuration.ConfigurationKeys; import gobblin.rest.JobExecutionInfoServer; +import gobblin.runtime.app.ServiceBasedAppLauncher; import gobblin.util.ConfigUtils; import gobblin.util.EmailUtils; import gobblin.util.ExecutorsUtils; @@ -290,7 +288,8 @@ public class GobblinYarnAppLauncher { services.add(executionInfoServer); if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) { LOGGER.info("Starting the admin UI server since it is enabled"); - services.add(new AdminWebServer(properties, executionInfoServer.getAdvertisedServerUri())); + services.add(ServiceBasedAppLauncher.createAdminServer(properties, + executionInfoServer.getAdvertisedServerUri())); } } else if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) { LOGGER.warn("NOT starting the admin UI because the job execution info server is NOT enabled"); @@ -379,7 +378,7 @@ public class GobblinYarnAppLauncher { @Subscribe public void handleGetApplicationReportFailureEvent( - @SuppressWarnings("unused") GetApplicationReportFailureEvent getApplicationReportFailureEvent) { + GetApplicationReportFailureEvent getApplicationReportFailureEvent) { int numConsecutiveFailures = this.getApplicationReportFailureCount.incrementAndGet(); if (numConsecutiveFailures > this.maxGetApplicationReportFailures) { LOGGER.warn(String
