AMBARI-18051 - Services should be able to provide their own pre-req checks by supplying a jar file
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/87423d64 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/87423d64 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/87423d64 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 87423d64f54d896c62d1a9245eb03a97763e35a4 Parents: 6d35228 Author: Tim Thorpe <[email protected]> Authored: Tue Sep 27 06:05:31 2016 -0700 Committer: Tim Thorpe <[email protected]> Committed: Tue Sep 27 06:05:31 2016 -0700 ---------------------------------------------------------------------- ambari-server/pom.xml | 34 +++ .../ambari/server/checks/CheckDescription.java | 5 +- .../server/checks/UpgradeCheckRegistry.java | 76 ++++++ .../PreUpgradeCheckResourceProvider.java | 27 +- .../server/stack/CommonServiceDirectory.java | 50 ++-- .../ambari/server/stack/ServiceDirectory.java | 112 +++++++- .../ambari/server/stack/ServiceModule.java | 4 + .../server/stack/StackServiceDirectory.java | 81 +++--- .../apache/ambari/server/state/ServiceInfo.java | 14 + .../PreUpgradeCheckResourceProviderTest.java | 255 +++++++++++++++++++ .../sample/checks/SampleServiceCheck.java | 35 +++ .../ambari/server/stack/ServiceModuleTest.java | 30 +++ .../server/stack/StackManagerExtensionTest.java | 7 + 13 files changed, 636 insertions(+), 94 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 5731c9d..354b6cb 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -576,6 +576,40 @@ </configuration> </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.0.2</version> + <executions> + <execution> + <id>create-sample-upgrade-check-jar</id> + <phase>process-test-classes</phase> + <goals> + <goal>test-jar</goal> + </goals> + <configuration> + <outputDirectory>target/test-classes/checks</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>generate-test-oozie2-checks-dir</id> + <phase>process-test-classes</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <mkdir dir="target/test-classes/extensions/EXT/0.1/services/OOZIE2/checks/tmp"/> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> <version>1.2.1</version> http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java index 498481d..7fdd0ce 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java @@ -309,9 +309,8 @@ public class CheckDescription { private PrereqCheckType m_type; private String m_description; private Map<String, String> m_fails; - public CheckDescription(String name, PrereqCheckType type, String description, - Map<String, String> fails) { - m_name = name; + public CheckDescription(String name, PrereqCheckType type, String description, Map<String, String> fails) { + m_name = name; m_type = type; m_description = description; m_fails = fails; http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckRegistry.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckRegistry.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckRegistry.java index 4ed345c..cecf6c5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckRegistry.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckRegistry.java @@ -17,14 +17,24 @@ */ package org.apache.ambari.server.checks; +import java.io.File; +import java.io.FilenameFilter; +import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import com.google.inject.Singleton; + +import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.stack.UpgradePack; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.ClassUtils; /** * The {@link UpgradeCheckRegistry} contains the ordered list of all pre-upgrade @@ -33,6 +43,7 @@ import org.apache.ambari.server.state.stack.UpgradePack; */ @Singleton public class UpgradeCheckRegistry { + private static Logger LOG = LoggerFactory.getLogger(UpgradeCheckRegistry.class); /** * The list of upgrade checks to run through. @@ -59,6 +70,71 @@ public class UpgradeCheckRegistry { return new ArrayList<AbstractCheckDescriptor>(m_upgradeChecks); } + public List<AbstractCheckDescriptor> getServiceLevelUpgradeChecks(UpgradePack upgradePack, Map<String, ServiceInfo> services) { + List<String> prerequisiteChecks = upgradePack.getPrerequisiteChecks(); + List<String> missingChecks = new ArrayList<String>(); + for (String prerequisiteCheck : prerequisiteChecks) { + if (!isRegistered(prerequisiteCheck)) { + missingChecks.add(prerequisiteCheck); + } + } + + List<AbstractCheckDescriptor> checks = new ArrayList<>(missingChecks.size()); + if (missingChecks.isEmpty()) { + return checks; + } + + List<URL> urls = new ArrayList<URL>(); + for (ServiceInfo service : services.values()) { + File dir = service.getChecksFolder(); + File[] jars = dir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar"); + } + }); + for (File jar : jars) { + try { + URL url = jar.toURI().toURL(); + urls.add(url); + LOG.debug("Adding service check jar to classpath: {}", url.toString()); + } + catch (Exception e) { + LOG.error("Failed to add service check jar to classpath: {}", jar.getAbsolutePath(), e); + } + } + } + + ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), ClassUtils.getDefaultClassLoader()); + for (String prerequisiteCheck : missingChecks) { + Class<?> clazz = null; + try { + clazz = ClassUtils.resolveClassName(prerequisiteCheck, classLoader); + } + catch (IllegalArgumentException illegalArgumentException) { + LOG.error("Unable to find upgrade check {}", prerequisiteCheck, illegalArgumentException); + } + try { + if (clazz != null) { + AbstractCheckDescriptor upgradeCheck = (AbstractCheckDescriptor) clazz.newInstance(); + checks.add(upgradeCheck); + } + } catch (Exception exception) { + LOG.error("Unable to create upgrade check {}", prerequisiteCheck, exception); + } + } + return checks; + } + + private boolean isRegistered(String prerequisiteCheck) { + for (AbstractCheckDescriptor descriptor: m_upgradeChecks){ + if (prerequisiteCheck.equals(descriptor.getClass().getName())){ + return true; + } + } + return false; + } + /** * Gets an ordered and filtered list of the upgrade checks. * @param upgradePack Upgrade pack object with the list of required checks to be included http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProvider.java index 7d7b618..7e54f83 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProvider.java @@ -44,21 +44,26 @@ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.CheckHelper; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.UpgradeHelper; import org.apache.ambari.server.state.stack.PrerequisiteCheck; +import org.apache.ambari.server.state.stack.UpgradePack; +import org.apache.ambari.server.state.stack.upgrade.Direction; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.google.inject.Provider; -import org.apache.ambari.server.state.stack.UpgradePack; -import org.apache.ambari.server.state.stack.upgrade.Direction; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; /** * Resource provider for pre-upgrade checks. */ @StaticallyInject public class PreUpgradeCheckResourceProvider extends ReadOnlyResourceProvider { + private static Logger LOG = LoggerFactory.getLogger(PreUpgradeCheckResourceProvider.class); //----- Property ID constants --------------------------------------------- @@ -89,6 +94,9 @@ public class PreUpgradeCheckResourceProvider extends ReadOnlyResourceProvider { @Inject private static Provider<UpgradeHelper> upgradeHelper; + @Inject + private static CheckHelper checkHelper; + private static Set<String> pkPropertyIds = Collections.singleton(UPGRADE_CHECK_ID_PROPERTY_ID); public static Set<String> propertyIds = Sets.newHashSet( @@ -113,9 +121,6 @@ public class PreUpgradeCheckResourceProvider extends ReadOnlyResourceProvider { } }; - @Inject - private static CheckHelper checkHelper; - /** * Constructor. * @@ -185,10 +190,18 @@ public class PreUpgradeCheckResourceProvider extends ReadOnlyResourceProvider { } // ToDo: properly handle exceptions, i.e. create fake check with error description - List<AbstractCheckDescriptor> upgradeChecksToRun = upgradeCheckRegistry.getFilteredUpgradeChecks(upgradePack); upgradeCheckRequest.setPrerequisiteCheckConfig(upgradePack.getPrerequisiteCheckConfig()); + try { + // Register all the custom prechecks from the services + Map<String, ServiceInfo> services = getManagementController().getAmbariMetaInfo().getServices(stackName, upgradePack.getTarget()); + List<AbstractCheckDescriptor> serviceLevelUpgradeChecksToRun = upgradeCheckRegistry.getServiceLevelUpgradeChecks(upgradePack, services); + upgradeChecksToRun.addAll(serviceLevelUpgradeChecksToRun); + } catch (AmbariException ambariException) { + LOG.error("Unable to register all the custom prechecks from the services", ambariException); + } + for (PrerequisiteCheck prerequisiteCheck : checkHelper.performChecks(upgradeCheckRequest, upgradeChecksToRun)) { final Resource resource = new ResourceImpl(Resource.Type.PreUpgradeCheck); setResourceProperty(resource, UPGRADE_CHECK_ID_PROPERTY_ID, prerequisiteCheck.getId(), requestedIds); http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java index cdedbb4..40e7105 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java @@ -19,8 +19,6 @@ package org.apache.ambari.server.stack; import org.apache.ambari.server.AmbariException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; @@ -28,10 +26,6 @@ import java.io.File; * Encapsulates IO operations on a common services directory. */ public class CommonServiceDirectory extends ServiceDirectory { - /** - * logger instance - */ - private static final Logger LOG = LoggerFactory.getLogger(CommonServiceDirectory.class); /** * Constructor. @@ -62,36 +56,30 @@ public class CommonServiceDirectory extends ServiceDirectory { @Override /** - * Calculate the common service directories - * packageDir Format: common-services/<serviceName>/<serviceVersion>/package - * Example: - * directory: "/var/lib/ambari-server/resources/common-services/HDFS/1.0" - * packageDir: "common-services/HDFS/1.0/package" + * @return the service name-version (will be used for logging purposes by superclass) */ - protected void calculateDirectories() { + public String getService() { File serviceVersionDir = new File(getAbsolutePath()); File serviceDir = serviceVersionDir.getParentFile(); - String serviceId = String.format("%s/%s", serviceDir.getName(), serviceVersionDir.getName()); + String service = String.format("%s-%s", serviceDir.getName(), serviceVersionDir.getName()); + return service; + } - File absPackageDir = new File(getAbsolutePath() + File.separator + PACKAGE_FOLDER_NAME); - if(absPackageDir.isDirectory()) { - packageDir = absPackageDir.getPath().substring(serviceDir.getParentFile().getParentFile().getPath().length() + 1); - LOG.debug(String.format("Service package folder for common service %s has been resolved to %s", - serviceId, packageDir)); - } else { - LOG.debug(String.format("Service package folder %s for common service %s does not exist.", - absPackageDir, serviceId )); - } + @Override + /** + * @return the resources directory + */ + protected File getResourcesDirectory() { + File serviceVersionDir = new File(getAbsolutePath()); + return serviceVersionDir.getParentFile().getParentFile().getParentFile(); + } - File absUpgradesDir = new File(getAbsolutePath() + File.separator + UPGRADES_FOLDER_NAME); - if(absUpgradesDir.isDirectory()) { - upgradesDir = absUpgradesDir; - LOG.debug(String.format("Service upgrades folder for common service %s has been resolved to %s", - serviceId, upgradesDir)); - } else { - LOG.debug(String.format("Service upgrades folder %s for common service %s does not exist.", - absUpgradesDir, serviceId )); - } + @Override + /** + * @return the text common-services (will be used for logging purposes by superclass) + */ + public String getStack() { + return "common-services"; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java index 30663a3..00dc046 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java @@ -88,6 +88,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { protected File upgradesDir; /** + * checks directory path + */ + protected File checksDir; + + /** * service metainfo file object representation */ private ServiceMetainfoXml metaInfoXml; @@ -108,6 +113,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { protected static final String UPGRADES_FOLDER_NAME = "upgrades"; /** + * checks directory name + */ + protected static final String CHECKS_FOLDER_NAME = "checks"; + + /** * service metainfo file name */ private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml"; @@ -153,6 +163,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { } /** + * Obtain the checks directory path. + * + * @return checks directory path + */ + public File getChecksDir() { + return checksDir; + } + + /** * Obtain the metrics file. * * @return metrics file @@ -235,7 +254,7 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { * Parse the service directory. */ protected void parsePath() throws AmbariException { - calculateDirectories(); + calculateDirectories(getStack(), getService()); parseMetaInfoFile(); File af = new File(directory, AmbariMetaInfo.SERVICE_ALERT_FILE_NAME); @@ -265,12 +284,101 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory { File themeFile = new File(directory, AmbariMetaInfo.SERVICE_THEME_FILE_NAME); this.themeFile = themeFile.exists() ? themeFile : null; + + File checksFile = new File(directory, AmbariMetaInfo.SERVICE_THEME_FILE_NAME); + this.themeFile = themeFile.exists() ? themeFile : null; } /** + * @return the service identifier required. ex: service name for stack services or the service/version for common services + */ + public abstract String getService(); + + /** + * @return the stack name/version or common-services + */ + public abstract String getStack(); + + /** * Calculate the service specific directories. */ - protected abstract void calculateDirectories(); + protected void calculateDirectories(String stack, String service) { + calculatePackageDirectory(stack, service); + calculateUpgradesDirectory(stack, service); + calculateChecksDirectory(stack, service); + } + + /** + * @param directoryName + * @param stack + * @param service + * @return the directory if it exists and is not empty + */ + protected File resolveDirectory(String directoryName, String stack, String service) { + File directory = new File(getAbsolutePath() + File.separator + directoryName); + if (directory.isDirectory()) { + String[] files = directory.list(); + int fileCount = files.length; + if (fileCount > 0) { + LOG.debug("Service {} folder for service {} in {} has been resolved to {}", directoryName, service, stack, directory); + return directory; + } + else { + LOG.debug("Service folder {} is empty.", directory); + } + } + else { + LOG.debug("Service folder {}does not exist.", directory); + } + return null; + } + + /** + * @param directoryName + * @param stack + * @param service + * @return the relative path of the directory if it exists and is not empty + */ + protected String resolveRelativeDirectoryPathString(File resourcesDir, String directoryName, String stack, String service) { + File dir = resolveDirectory(directoryName, stack, service); + if (dir != null) { + return dir.getPath().substring(resourcesDir.getPath().length() + 1); + } + return null; + } + + /** + * @return the resources directory + */ + protected abstract File getResourcesDirectory(); + + /** + * Sets the packageDir if the path exists and is not empty + * @param stack + * @param service + */ + protected void calculatePackageDirectory(String stack, String service) { + packageDir = resolveRelativeDirectoryPathString(getResourcesDirectory(), PACKAGE_FOLDER_NAME, stack, service); + + } + + /** + * Sets the upgradesDir if the dir exists and is not empty + * @param stack + * @param service + */ + protected void calculateUpgradesDirectory(String stack, String service) { + upgradesDir = resolveDirectory(UPGRADES_FOLDER_NAME, stack, service); + } + + /** + * Sets the checksDir if the dir exists and is not empty + * @param stack + * @param service + */ + protected void calculateChecksDirectory(String stack, String service) { + checksDir = resolveDirectory(CHECKS_FOLDER_NAME, stack, service); + } /** * Unmarshal the metainfo file into its object representation. http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java index a77a22f..650bdf1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java @@ -142,6 +142,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem serviceInfo.setSchemaVersion(AmbariMetaInfo.SCHEMA_VERSION_2); serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir()); serviceInfo.setServiceUpgradesFolder(serviceDirectory.getUpgradesDir()); + serviceInfo.setChecksFolder(serviceDirectory.getChecksDir()); serviceInfo.setAdvisorFile(serviceDirectory.getAdvisorFile()); serviceInfo.setAdvisorName(serviceDirectory.getAdvisorName(serviceInfo.getName())); @@ -253,6 +254,9 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem if (serviceInfo.getRoleCommandOrder() == null) { serviceInfo.setRoleCommandOrder(parent.getRoleCommandOrder()); } + if (serviceInfo.getChecksFolder() == null) { + serviceInfo.setChecksFolder(parent.getChecksFolder()); + } mergeCustomCommands(parent.getCustomCommands(), serviceInfo.getCustomCommands()); mergeConfigDependencies(parent); http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java index 8656896..611b6bd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java @@ -28,12 +28,19 @@ import org.apache.ambari.server.state.stack.RepositoryXml; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Encapsulates IO operations on a stack service directory. */ public class StackServiceDirectory extends ServiceDirectory { /** + * logger instance + */ + private static final Logger LOG = LoggerFactory.getLogger(StackServiceDirectory.class); + + + /** * repository file */ @Nullable @@ -45,12 +52,6 @@ public class StackServiceDirectory extends ServiceDirectory { @Nullable private String repoDir; - - /** - * logger instance - */ - private static final Logger LOG = LoggerFactory.getLogger(StackServiceDirectory.class); - /** * Constructor. * @@ -61,7 +62,6 @@ public class StackServiceDirectory extends ServiceDirectory { super(servicePath); } - /** * Obtain the repository xml file if exists or null * @@ -82,7 +82,6 @@ public class StackServiceDirectory extends ServiceDirectory { return repoDir; } - @Override /** * Obtain the advisor name. @@ -129,55 +128,35 @@ public class StackServiceDirectory extends ServiceDirectory { @Override /** - * Calculate the stack service directories. - * packageDir Format: stacks/<stackName>/<stackVersion>/services/<serviceName>/package - * Example: - * directory: "/var/lib/ambari-server/resources/stacks/HDP/2.0.6/services/HDFS" - * packageDir: "stacks/HDP/2.0.6/services/HDFS/package" + * @return the resources directory */ - protected void calculateDirectories() { + protected File getResourcesDirectory() { File serviceDir = new File(getAbsolutePath()); - File stackVersionDir = serviceDir.getParentFile().getParentFile(); - File stackDir = stackVersionDir.getParentFile(); + return serviceDir.getParentFile().getParentFile().getParentFile().getParentFile().getParentFile(); + } - String stackId = String.format("%s-%s", stackDir.getName(), stackVersionDir.getName()); - File absPackageDir = new File(getAbsolutePath() + File.separator + PACKAGE_FOLDER_NAME); - if (absPackageDir.isDirectory()) { - String[] files = absPackageDir.list(); - int fileCount = files.length; - if (fileCount > 0) { - packageDir = absPackageDir.getPath().substring(stackDir.getParentFile().getParentFile().getPath().length() + 1); - LOG.debug("Service package folder for service {} for stack {} has been resolved to {}", - serviceDir.getName(), stackId, packageDir); - } - else { - LOG.debug("Service package folder {} for service {} for stack {} is empty.", - absPackageDir, serviceDir.getName(), stackId); - } - } else { - LOG.debug("Service package folder {} for service {} for stack {} does not exist.", - absPackageDir, serviceDir.getName(), stackId); - } + @Override + /** + * @return the service name (will be used for logging purposes by superclass) + */ + public String getService() { + File serviceDir = new File(getAbsolutePath()); - File absUpgradesDir = new File(getAbsolutePath() + File.separator + UPGRADES_FOLDER_NAME); - if (absUpgradesDir.isDirectory()) { - String[] files = absUpgradesDir.list(); - int fileCount = files.length; - if (fileCount > 0) { - upgradesDir = absUpgradesDir; - LOG.debug("Service upgrades folder for service {} for stack {} has been resolved to {}", - serviceDir.getName(), stackId, packageDir); - } - else { - LOG.debug("Service upgrades folder {} for service {} for stack {} is empty.", - absUpgradesDir, serviceDir.getName(), stackId); - } - } else { - LOG.debug("Service upgrades folder {} for service {} for stack {} does not exist.", - absUpgradesDir, serviceDir.getName(), stackId); - } + return serviceDir.getName(); } + @Override + /** + * @return the stack name-version (will be used for logging purposes by superclass) + */ + public String getStack() { + File serviceDir = new File(getAbsolutePath()); + File stackVersionDir = serviceDir.getParentFile().getParentFile(); + File stackDir = stackVersionDir.getParentFile(); + + String stackId = String.format("%s-%s", stackDir.getName(), stackVersionDir.getName()); + return stackId; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java index 6fda8bc..b0d81c3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java @@ -270,6 +270,12 @@ public class ServiceInfo implements Validable{ @XmlTransient private File serviceUpgradesFolder; + /** + * Stores the path to the checks folder which contains prereq check jars for the given service. + */ + @XmlTransient + private File checksFolder; + public boolean isDeleted() { return isDeleted; } @@ -600,6 +606,14 @@ public String getVersion() { this.serviceUpgradesFolder = serviceUpgradesFolder; } + public File getChecksFolder() { + return checksFolder; + } + + public void setChecksFolder(File checksFolder) { + this.checksFolder = checksFolder; + } + /** * Exposes (and initializes on first use) map of os-specific details. * @return map of OS specific details keyed by family http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java new file mode 100644 index 0000000..6a0fa12 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.ActionDBAccessor; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.checks.AbstractCheckDescriptor; +import org.apache.ambari.server.checks.UpgradeCheckRegistry; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.controller.MaintenanceStateHelper; +import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.utilities.PredicateBuilder; +import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.events.jpa.EntityManagerCacheInvalidationEvent; +import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.scheduler.ExecutionScheduler; +import org.apache.ambari.server.stack.StackManagerFactory; +import org.apache.ambari.server.state.CheckHelper; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Host; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceFactory; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.UpgradeHelper; +import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrereqCheckType; +import org.apache.ambari.server.state.stack.UpgradePack; +import org.apache.ambari.server.state.stack.UpgradePack.PrerequisiteCheckConfig; +import org.apache.ambari.server.state.stack.upgrade.Direction; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.persistence.EntityManager; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; + +import static org.easymock.EasyMock.anyBoolean; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.isNull; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +/** + * PreUpgradeCheckResourceProvider tests. + */ +public class PreUpgradeCheckResourceProviderTest { + + @Test + public void testGetResources() throws Exception{ + Injector injector = createInjector(); + AmbariManagementController managementController = injector.getInstance(AmbariManagementController.class); + + Clusters clusters = injector.getInstance(Clusters.class); + UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class); + + RepositoryVersionDAO repoDao = injector.getInstance(RepositoryVersionDAO.class); + RepositoryVersionEntity repo = createNiceMock(RepositoryVersionEntity.class); + UpgradePack upgradePack = createNiceMock(UpgradePack.class); + PrerequisiteCheckConfig config = createNiceMock(PrerequisiteCheckConfig.class); + + Cluster cluster = createNiceMock(Cluster.class); + Service service = createNiceMock(Service.class); + ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class); + + StackId currentStackId = createNiceMock(StackId.class); + StackId targetStackId = createNiceMock(StackId.class); + ServiceFactory serviceFactory = createNiceMock(ServiceFactory.class); + AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class); + + Map<String, Service> allServiceMap = new HashMap<String, Service>(); + allServiceMap.put("Service100", service); + Map<String, ServiceInfo> allServiceInfoMap = new HashMap<String, ServiceInfo>(); + allServiceInfoMap.put("Service100", serviceInfo); + + // set expectations + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getServiceFactory()).andReturn(serviceFactory).anyTimes(); + + expect(clusters.getCluster("Cluster100")).andReturn(cluster).anyTimes(); + expect(cluster.getServices()).andReturn(allServiceMap).anyTimes(); + expect(cluster.getService("Service100")).andReturn(service).anyTimes(); + expect(cluster.getCurrentStackVersion()).andReturn(currentStackId).anyTimes(); + + expect(currentStackId.getStackName()).andReturn("Stack100").anyTimes(); + expect(currentStackId.getStackVersion()).andReturn("1.0").anyTimes(); + expect(targetStackId.getStackName()).andReturn("Stack100").anyTimes(); + expect(targetStackId.getStackVersion()).andReturn("1.1").anyTimes(); + + expect(repoDao.findByStackNameAndVersion("Stack100", "Repo100")).andReturn(repo).anyTimes(); + expect(repo.getStackId()).andReturn(targetStackId).atLeastOnce(); + expect(upgradeHelper.suggestUpgradePack("Cluster100", "1.0", "Repo100", Direction.UPGRADE, UpgradeType.NON_ROLLING, "upgrade_pack11")).andReturn(upgradePack); + + List<AbstractCheckDescriptor> upgradeChecksToRun = new LinkedList<AbstractCheckDescriptor>(); + List<String> prerequisiteChecks = new LinkedList<String>(); + prerequisiteChecks.add("org.apache.ambari.server.sample.checks.SampleServiceCheck"); + expect(upgradePack.getPrerequisiteCheckConfig()).andReturn(config); + expect(upgradePack.getPrerequisiteChecks()).andReturn(prerequisiteChecks).anyTimes(); + expect(upgradePack.getTarget()).andReturn("1.1").anyTimes(); + + expect(ambariMetaInfo.getServices("Stack100", "1.1")).andReturn(allServiceInfoMap).anyTimes(); + String checks = ClassLoader.getSystemClassLoader().getResource("checks").getPath(); + expect(serviceInfo.getChecksFolder()).andReturn(new File(checks)); + + // replay + replay(managementController, clusters, cluster, service, serviceInfo, repoDao, repo, upgradeHelper, + ambariMetaInfo, upgradePack, config, currentStackId, targetStackId, serviceFactory); + + ResourceProvider provider = getPreUpgradeCheckResourceProvider(managementController, injector); + // create the request + Request request = PropertyHelper.getReadRequest(new HashSet<String>()); + PredicateBuilder builder = new PredicateBuilder(); + Predicate predicate = builder.property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_CLUSTER_NAME_PROPERTY_ID).equals("Cluster100").and() + .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_PACK_PROPERTY_ID).equals("upgrade_pack11").and() + .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_TYPE_PROPERTY_ID).equals(UpgradeType.NON_ROLLING).and() + .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_REPOSITORY_VERSION_PROPERTY_ID).equals("Repo100").toPredicate(); + + + System.out.println("PreUpgradeCheckResourceProvider - " + provider); + Set<Resource> resources = Collections.emptySet(); + try { + resources = provider.getResources(request, predicate); + } + catch (Exception e) { + e.printStackTrace(); + } + + Assert.assertEquals(1, resources.size()); + for (Resource resource : resources) { + String id = (String) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_ID_PROPERTY_ID); + Assert.assertEquals("SAMPLE_SERVICE_CHECK", id); + String description = (String) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_CHECK_PROPERTY_ID); + Assert.assertEquals("Sample service check description.", description); + PrereqCheckStatus status = (PrereqCheckStatus) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_STATUS_PROPERTY_ID); + Assert.assertEquals(PrereqCheckStatus.FAIL, status); + String reason = (String) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_REASON_PROPERTY_ID); + Assert.assertEquals("Sample service check always fails.", reason); + PrereqCheckType checkType = (PrereqCheckType) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_CHECK_TYPE_PROPERTY_ID); + Assert.assertEquals(PrereqCheckType.HOST, checkType); + String clusterName = (String) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_CLUSTER_NAME_PROPERTY_ID); + Assert.assertEquals("Cluster100", clusterName); + UpgradeType upgradeType = (UpgradeType) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_TYPE_PROPERTY_ID); + Assert.assertEquals(UpgradeType.NON_ROLLING, upgradeType); + } + + // verify + verify(managementController, clusters, cluster, service, serviceInfo, repoDao, repo, upgradeHelper, + ambariMetaInfo, upgradePack, config, currentStackId, targetStackId, serviceFactory); + } + + /** + * This factory method creates PreUpgradeCheckResourceProvider using the mock managementController + */ + public PreUpgradeCheckResourceProvider getPreUpgradeCheckResourceProvider(AmbariManagementController managementController, Injector injector) throws AmbariException { + //UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class); + //injector.injectMembers(upgradeHelper); + PreUpgradeCheckResourceProvider provider = new PreUpgradeCheckResourceProvider(managementController); + return provider; + } + + static class TestClustersProvider implements Provider<Clusters> { + private static Clusters clusters = createNiceMock(Clusters.class); + + @Override + public Clusters get() { + return clusters; + } + } + + static class TestUpgradeHelperProvider implements Provider<UpgradeHelper> { + private static UpgradeHelper upgradeHelper = createNiceMock(UpgradeHelper.class); + + @Override + public UpgradeHelper get() { + return upgradeHelper; + } + } + + private Injector createInjector() throws Exception { + return Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + Provider<Clusters> clustersProvider = new TestClustersProvider(); + Provider<UpgradeHelper> upgradeHelperProvider = new TestUpgradeHelperProvider(); + CheckHelper checkHelper = new CheckHelper(); + UpgradeCheckRegistry registry = new UpgradeCheckRegistry(); + + bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class)); + bind(CheckHelper.class).toInstance(checkHelper); + bind(Clusters.class).toProvider(TestClustersProvider.class); + bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + bind(EntityManager.class).toInstance(createNiceMock(EntityManager.class)); + bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + bind(RepositoryVersionDAO.class).toInstance(createNiceMock(RepositoryVersionDAO.class)); + bind(StackManagerFactory.class).toInstance(createNiceMock(StackManagerFactory.class)); + bind(UpgradeCheckRegistry.class).toInstance(registry); + bind(UpgradeHelper.class).toProvider(TestUpgradeHelperProvider.class); + + requestStaticInjection(PreUpgradeCheckResourceProvider.class); + } + }); + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/test/java/org/apache/ambari/server/sample/checks/SampleServiceCheck.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/sample/checks/SampleServiceCheck.java b/ambari-server/src/test/java/org/apache/ambari/server/sample/checks/SampleServiceCheck.java new file mode 100644 index 0000000..c91793e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/sample/checks/SampleServiceCheck.java @@ -0,0 +1,35 @@ +package org.apache.ambari.server.sample.checks; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.checks.AbstractCheckDescriptor; +import org.apache.ambari.server.checks.CheckDescription; +import org.apache.ambari.server.controller.PrereqCheckRequest; +import org.apache.ambari.server.state.stack.PrereqCheckStatus; +import org.apache.ambari.server.state.stack.PrereqCheckType; +import org.apache.ambari.server.state.stack.PrerequisiteCheck; + +import com.google.common.collect.ImmutableMap; + +public class SampleServiceCheck extends AbstractCheckDescriptor { + + public SampleServiceCheck() { + super(new CheckDescription("SAMPLE_SERVICE_CHECK", + PrereqCheckType.HOST, + "Sample service check description.", + new ImmutableMap.Builder<String, String>() + .put(AbstractCheckDescriptor.DEFAULT, + "Sample service check default property description.").build())); + } + + @Override + public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException { + prerequisiteCheck.setFailReason("Sample service check always fails."); + prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL); + } + + @Override + public boolean isStackUpgradeAllowedToBypassPreChecks() { + return false; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java index 304fd5c..a9a8fdb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java @@ -437,6 +437,36 @@ public class ServiceModuleTest { } @Test + public void testResolve_UpgradeCheckDirectory() throws Exception { + File checks = new File("checks"); + + // check directory specified in child only + ServiceInfo info = new ServiceInfo(); + ServiceInfo parentInfo = new ServiceInfo(); + ServiceModule child = createServiceModule(info); + ServiceModule parent = createServiceModule(parentInfo); + child.getModuleInfo().setChecksFolder(checks); + resolveService(child, parent); + assertEquals(checks.getPath(), child.getModuleInfo().getChecksFolder().getPath()); + + // check directory specified in parent only + child = createServiceModule(info); + parent = createServiceModule(parentInfo); + parent.getModuleInfo().setChecksFolder(checks); + resolveService(child, parent); + assertEquals(checks.getPath(), child.getModuleInfo().getChecksFolder().getPath()); + + // check directory set in both + info.setChecksFolder(checks); + child = createServiceModule(info); + child.getModuleInfo().setChecksFolder(checks); + parent = createServiceModule(parentInfo); + parent.getModuleInfo().setChecksFolder(new File("other")); + resolveService(child, parent); + assertEquals(checks.getPath(), child.getModuleInfo().getChecksFolder().getPath()); + } + + @Test public void testResolve_CustomCommands() throws Exception { List<CustomCommandDefinition> customCommands = new ArrayList<CustomCommandDefinition>(); CustomCommandDefinition cmd1 = new CustomCommandDefinition(); http://git-wip-us.apache.org/repos/asf/ambari/blob/87423d64/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java index 659ae12..044f2c4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java @@ -29,6 +29,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; +import java.io.FilenameFilter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -105,6 +106,9 @@ public class StackManagerExtensionTest { assertNotNull("Package dir is " + oozie.getServicePackageFolder(), oozie.getServicePackageFolder()); assertTrue("Package dir is " + oozie.getServicePackageFolder(), oozie.getServicePackageFolder().contains("extensions/EXT/0.1/services/OOZIE2/package")); assertEquals(oozie.getVersion(), "3.2.0"); + File checks = oozie.getChecksFolder(); + assertNotNull(checks); + assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks")); extension = stackManager.getExtension("EXT", "0.2"); assertNotNull("EXT 0.2's parent: " + extension.getParentExtensionVersion(), extension.getParentExtensionVersion()); @@ -114,6 +118,9 @@ public class StackManagerExtensionTest { assertNotNull("Package dir is " + oozie.getServicePackageFolder(), oozie.getServicePackageFolder()); assertTrue("Package dir is " + oozie.getServicePackageFolder(), oozie.getServicePackageFolder().contains("extensions/EXT/0.1/services/OOZIE2/package")); assertEquals(oozie.getVersion(), "4.0.0"); + checks = oozie.getChecksFolder(); + assertNotNull(checks); + assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks")); StackInfo stack = stackManager.getStack("HDP", "0.2"); assertNotNull(stack.getService("OOZIE2"));
