Repository: ambari Updated Branches: refs/heads/branch-2.6 17f89c01f -> eecd8513a
AMBARI-22337 each service should be able to implement server actions, package them add a jar to be loaded during EU (port from trunk to branch-26)(dili) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eecd8513 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eecd8513 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eecd8513 Branch: refs/heads/branch-2.6 Commit: eecd8513a304641617cc8602f5805eee8aaf3c69 Parents: 17f89c0 Author: Di Li <d...@apache.org> Authored: Mon Nov 20 13:54:31 2017 -0500 Committer: Di Li <d...@apache.org> Committed: Mon Nov 20 13:54:31 2017 -0500 ---------------------------------------------------------------------- ambari-server/pom.xml | 12 ++ .../serveraction/ServerActionExecutor.java | 147 +++++++++++++++++-- .../ambari/server/stack/ServiceDirectory.java | 29 ++++ .../ambari/server/stack/ServiceModule.java | 8 + .../apache/ambari/server/state/ServiceInfo.java | 14 ++ .../ambari/server/stack/ServiceModuleTest.java | 30 ++++ .../server/stack/StackManagerExtensionTest.java | 6 + 7 files changed, 232 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 342b3c0..eece739 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -165,6 +165,18 @@ </target> </configuration> </execution> + <execution> + <id>generate-test-oozie2-server-actions-dir</id> + <phase>process-test-classes</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <mkdir dir="target/test-classes/extensions/EXT/0.1/services/OOZIE2/server_actions/tmp"/> + </target> + </configuration> + </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java index 50e3cfe..e219dc3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java @@ -18,10 +18,17 @@ package org.apache.ambari.server.serveraction; +import java.io.File; +import java.io.FilenameFilter; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringTokenizer; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; @@ -38,11 +45,17 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.UpgradeContext.UpgradeServiceSummary; +import org.apache.ambari.server.state.UpgradeContext.UpgradeSummary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.util.ClassUtils; import com.google.inject.Inject; import com.google.inject.Injector; @@ -495,7 +508,6 @@ public class ServerActionExecutor { throw new AmbariException("Missing ExecutionCommand data"); } else { Map<String, String> roleParams = executionCommand.getRoleParams(); - if (roleParams == null) { throw new AmbariException("Missing RoleParams data"); } else { @@ -504,8 +516,30 @@ public class ServerActionExecutor { if (actionClassname == null) { throw new AmbariException("Missing action classname for server action"); } else { - ServerAction action = createServerAction(actionClassname); - + Map<String, ServiceInfo> services = new HashMap<String, ServiceInfo>(); + UpgradeSummary upgradeSummary = executionCommand.getUpgradeSummary(); + if (upgradeSummary != null) { + Map<String, UpgradeServiceSummary> upgradeServiceSummaries = upgradeSummary.services; + LOG.debug("UpgradeServiceSummary: " + upgradeServiceSummaries); + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + AmbariMetaInfo ambariMetaInfo = ambariManagementController.getAmbariMetaInfo(); + String serviceName = executionCommand.getServiceName(); + if (serviceName != null && !serviceName.isEmpty()){ + LOG.info(String.format("Server action %s is associated with service %s", actionClassname, serviceName)); + //Execution stage of a given service, only need to examine stack information for this one service + UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(serviceName); + addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, serviceName); + } else { + LOG.info(String.format("Server action %s is not associated with a service", actionClassname)); + //Load all Jars + for(String key: upgradeServiceSummaries.keySet()){ + UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(key); + addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, key); + } + } + LOG.info(String.format("Attempt to load server action classes from %s", services.keySet().toString())); + } + ServerAction action = createServerAction(actionClassname, services); if (action == null) { throw new AmbariException("Failed to create server action: " + actionClassname); } else { @@ -520,6 +554,30 @@ public class ServerActionExecutor { } } + private void addServiceInfo(Map<String, ServiceInfo> services, AmbariMetaInfo ambariMetaInfo, String stackId, String serviceName) { + List<String> stackInfo = getStackInfo(stackId); + LOG.debug(String.format("Stack info list: %s", stackInfo)); + if (stackInfo.size() > 1) { + try { + ServiceInfo service = ambariMetaInfo.getService(stackInfo.get(0), stackInfo.get(1), serviceName); + LOG.debug(String.format("Adding %s to the list of services for loading external Jars...", service.getName())); + services.put(serviceName, service); + } catch (AmbariException e) { + LOG.error(String.format("Failed to obtain service info for stack %s, service name %s", stackId, serviceName), e); + } + } + } + + private List<String> getStackInfo(String stackId) { + LOG.debug(String.format("Stack id: %s", stackId)); + StringTokenizer tokens = new StringTokenizer(stackId, "-"); + List<String> info = new ArrayList<String>(); + while (tokens.hasMoreElements()) { + info.add((String)tokens.nextElement()); + } + return info; + } + /** * Attempts to create an instance of the ServerAction class implementation specified in * classname. @@ -528,24 +586,85 @@ public class ServerActionExecutor { * @return the instantiated ServerAction implementation * @throws AmbariException */ - private ServerAction createServerAction(String classname) throws AmbariException { - try { - Class<?> actionClass = Class.forName(classname); + private ServerAction createServerAction(String classname, Map<String, ServiceInfo> services) throws AmbariException { + Class<?> actionClass = null; + actionClass = getServerActionClass(classname); + if (actionClass == null) { + LOG.debug(String.format("Did not find %s in Ambari, try to load it from external directories", classname)); + actionClass = getServiceLevelServerActionClass(classname, services); + } - if (actionClass == null) { - throw new AmbariException("Unable to load server action class: " + classname); + if (actionClass == null) { + throw new AmbariException("Unable to load server action class: " + classname); + } else { + LOG.debug(String.format("Ready to init server action %s", classname)); + Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class); + if (serverActionClass == null) { + throw new AmbariException("Unable to execute server action class, invalid type: " + classname); } else { - Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class); + return injector.getInstance(serverActionClass); + } + } + } - if (serverActionClass == null) { - throw new AmbariException("Unable to execute server action class, invalid type: " + classname); - } else { - return injector.getInstance(serverActionClass); + /** + * Load server action classes defined in the service level Jar files + * */ + private Class<?> getServiceLevelServerActionClass(String classname, Map<String, ServiceInfo> services) { + List<URL> urls = new ArrayList<>(); + for (ServiceInfo service : services.values()) { + LOG.debug(String.format("Checking service %s", service)); + File dir = service.getServerActionsFolder(); + if ( dir != null) { + LOG.debug(String.format("Service %s, external dir %s",service.getName(), dir.getAbsolutePath())); + File[] jars = dir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + LOG.debug(String.format("Checking folder %s", name)); + return name.endsWith(".jar"); + } + }); + for (File jar : jars) { + try { + URL url = jar.toURI().toURL(); + urls.add(url); + LOG.info("Adding server action jar to classpath: {}", url); + } + catch (Exception e) { + LOG.error("Failed to add server action jar to classpath: {}", jar.getAbsolutePath(), e); + } } + } else { + LOG.error(String.format("%s service server actions folder returned null", service)); + } + } + + ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), ClassUtils.getDefaultClassLoader()); + Class<?> actionClass = null; + try { + actionClass = ClassUtils.resolveClassName(classname, classLoader); + LOG.debug(String.format("Found external server action %s", classname)); + } catch(IllegalArgumentException illegalArgumentException) { + LOG.error(String.format("Unable to find server action %s in external server action directories", classname), illegalArgumentException); + } + + return actionClass; + } + + /** + * Load server action classes defined in Ambari source code + * */ + private Class<?> getServerActionClass(String classname) throws AmbariException{ + Class<?> actionClass = null; + try { + actionClass = Class.forName(classname); + if (actionClass == null) { + LOG.warn(String.format("Unable to load server action class: %s from Ambari", classname)); } } catch (ClassNotFoundException e) { - throw new AmbariException("Unable to load server action class: " + classname, e); + LOG.error(String.format("Unable to load server action class: %s", classname), e); } + return actionClass; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java index abef459..8af0c9a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java @@ -92,6 +92,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { protected File checksDir; /** + * server side action directory path + */ + protected File serverActionsDir; + + /** * service metainfo file object representation */ private ServiceMetainfoXml metaInfoXml; @@ -117,6 +122,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { protected static final String CHECKS_FOLDER_NAME = "checks"; /** + * Server actions directory name + */ + protected static final String SERVER_ACTIONS_FOLDER_NAME = "server_actions"; + + /** * service metainfo file name */ private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml"; @@ -171,6 +181,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { } /** + * Obtain the server side actions directory path. + * + * @return server side actions directory path + */ + public File getServerActionsDir() { + return serverActionsDir; + } + + /** * Obtain the metrics file. * * @return metrics file @@ -302,6 +321,7 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { calculatePackageDirectory(stack, service); calculateUpgradesDirectory(stack, service); calculateChecksDirectory(stack, service); + calculateServerActionsDirectory(stack, service); } /** @@ -377,6 +397,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { } /** + * Sets the serverActionsDir if the dir exists and is not empty + * @param stack + * @param service + */ + protected void calculateServerActionsDirectory(String stack, String service) { + serverActionsDir = resolveDirectory(SERVER_ACTIONS_FOLDER_NAME, stack, service); + } + + /** * Unmarshal the metainfo file into its object representation. * * @throws AmbariException if the metainfo file doesn't exist or http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java index 6699e0e..7a8fa64 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java @@ -144,6 +144,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir()); serviceInfo.setServiceUpgradesFolder(serviceDirectory.getUpgradesDir()); serviceInfo.setChecksFolder(serviceDirectory.getChecksDir()); + serviceInfo.setServerActionsFolder(serviceDirectory.getServerActionsDir()); serviceInfo.setAdvisorFile(serviceDirectory.getAdvisorFile()); serviceInfo.setAdvisorName(serviceDirectory.getAdvisorName(serviceInfo.getName())); @@ -259,6 +260,13 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem serviceInfo.setChecksFolder(parent.getChecksFolder()); } + /* + * Use parent's server actions if the current one does not have any. + */ + if (serviceInfo.getServerActionsFolder() == null) { + serviceInfo.setServerActionsFolder(parent.getServerActionsFolder()); + } + /** * If current stack version does not specify the credential store information * for the service, then use parent definition. http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java index b7afe53..a5ea449 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java @@ -282,6 +282,12 @@ public class ServiceInfo implements Validable{ @XmlTransient private File checksFolder; + /** + * Stores the path to the server actions folder which contains server actions jars for the given service. + */ + @XmlTransient + private File serverActionsFolder; + public boolean isDeleted() { return isDeleted; } @@ -744,6 +750,14 @@ public String getVersion() { this.checksFolder = checksFolder; } + public File getServerActionsFolder() { + return serverActionsFolder; + } + + public void setServerActionsFolder(File serverActionsFolder) { + this.serverActionsFolder = serverActionsFolder; + } + /** * Exposes (and initializes on first use) map of os-specific details. * @return map of OS specific details keyed by family http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java index 5efbc89..f8bfac8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java @@ -467,6 +467,36 @@ public class ServiceModuleTest { } @Test + public void testResolve_ServerActionDirectory() throws Exception { + File serverActions = new File("server_actions"); + + // check directory specified in child only + ServiceInfo info = new ServiceInfo(); + ServiceInfo parentInfo = new ServiceInfo(); + ServiceModule child = createServiceModule(info); + ServiceModule parent = createServiceModule(parentInfo); + child.getModuleInfo().setServerActionsFolder(serverActions); + resolveService(child, parent); + assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath()); + + // check directory specified in parent only + child = createServiceModule(info); + parent = createServiceModule(parentInfo); + parent.getModuleInfo().setServerActionsFolder(serverActions); + resolveService(child, parent); + assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath()); + + // check directory set in both + info.setServerActionsFolder(serverActions); + child = createServiceModule(info); + child.getModuleInfo().setServerActionsFolder(serverActions); + parent = createServiceModule(parentInfo); + parent.getModuleInfo().setServerActionsFolder(new File("other")); + resolveService(child, parent); + assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath()); + } + + @Test public void testResolve_CustomCommands() throws Exception { List<CustomCommandDefinition> customCommands = new ArrayList<CustomCommandDefinition>(); CustomCommandDefinition cmd1 = new CustomCommandDefinition(); http://git-wip-us.apache.org/repos/asf/ambari/blob/eecd8513/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java index 0676568..8165398 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java @@ -110,6 +110,9 @@ public class StackManagerExtensionTest { File checks = oozie.getChecksFolder(); assertNotNull(checks); assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks")); + File serverActions = oozie.getServerActionsFolder(); + assertNotNull(serverActions); + assertTrue("Server actions dir is " + serverActions.getPath(), serverActions.getPath().contains("extensions/EXT/0.1/services/OOZIE2/server_actions")); List<ThemeInfo> themes = oozie.getThemes(); assertNotNull(themes); assertTrue("Number of themes is " + themes.size(), themes.size() == 1); @@ -127,6 +130,9 @@ public class StackManagerExtensionTest { checks = oozie.getChecksFolder(); assertNotNull(checks); assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks")); + serverActions = oozie.getServerActionsFolder(); + assertNotNull(serverActions); + assertTrue("Server actions dir is " + serverActions.getPath(), serverActions.getPath().contains("extensions/EXT/0.1/services/OOZIE2/server_actions")); themes = oozie.getThemes(); assertNotNull(themes); assertTrue("Number of themes is " + themes.size(), themes.size() == 0);
