Repository: ambari Updated Branches: refs/heads/branch-2.6 596b145db -> 98be9abf8
AMBARI-21756. Use latest-vdf for default when version is unspecified (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/98be9abf Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/98be9abf Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/98be9abf Branch: refs/heads/branch-2.6 Commit: 98be9abf850fd04f1399c4894277b1770a018a5a Parents: 596b145 Author: Nate Cole <[email protected]> Authored: Fri Aug 18 11:54:00 2017 -0400 Committer: Nate Cole <[email protected]> Committed: Mon Aug 21 09:52:05 2017 -0400 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 9 ++- .../VersionDefinitionResourceProvider.java | 6 +- .../apache/ambari/server/state/StackInfo.java | 39 +++++++--- .../state/repository/VersionDefinitionXml.java | 7 ++ .../server/state/stack/LatestRepoCallable.java | 80 +++++++++++++++----- .../ambari/server/topology/AmbariContext.java | 48 +++++++++++- .../server/api/services/AmbariMetaInfoTest.java | 30 ++++++++ .../AmbariManagementControllerTest.java | 2 +- .../VersionDefinitionResourceProviderTest.java | 14 +--- .../ambari/server/stack/StackManagerTest.java | 14 ++-- .../server/topology/AmbariContextTest.java | 39 ++++++++-- .../resources/stacks/HDP/2.2.0/repos/hdp.json | 4 + .../stacks/HDP/2.2.0/repos/repoinfo.xml | 8 ++ .../stacks/HDP/2.2.0/repos/version-2.2.0.5.xml | 51 +++++++++++++ .../resources/stacks/HDP/2.2.1/metainfo.xml | 24 ++++++ .../resources/stacks/HDP/2.2.1/repos/hdp.json | 7 ++ .../stacks/HDP/2.2.1/repos/repoinfo.xml | 36 +++++++++ .../HDP/2.2.1/services/RANGER/alerts.json | 74 ++++++++++++++++++ 18 files changed, 430 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 2cecfb6..9afba8c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -1451,7 +1451,14 @@ public class AmbariMetaInfo { if (stack.isActive() && stack.isValid()) { try { - VersionDefinitionXml xml = VersionDefinitionXml.build(stack); + // !!! check for a "latest-vdf" one. This will be used for the default if one is not found. + VersionDefinitionXml xml = stack.getLatestVersionDefinition(); + + if (null == xml) { + // !!! "latest-vdf" was not found, use the stack. this is the last-ditch effort + xml = VersionDefinitionXml.build(stack); + } + versionDefinitions.put(String.format("%s-%s", stack.getName(), stack.getVersion()), xml); } catch (Exception e) { LOG.warn("Could not make a stack VDF for {}-{}: {}", http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java index ea592e5..5fc96d1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java @@ -91,10 +91,10 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc public static final String VERSION_DEF_STACK_NAME = "VersionDefinition/stack_name"; public static final String VERSION_DEF_STACK_VERSION = "VersionDefinition/stack_version"; - protected static final String VERSION_DEF_ID = "VersionDefinition/id"; + public static final String VERSION_DEF_ID = "VersionDefinition/id"; protected static final String VERSION_DEF_TYPE_PROPERTY_ID = "VersionDefinition/type"; protected static final String VERSION_DEF_DEFINITION_URL = "VersionDefinition/version_url"; - protected static final String VERSION_DEF_AVAILABLE_DEFINITION = "VersionDefinition/available"; + public static final String VERSION_DEF_AVAILABLE_DEFINITION = "VersionDefinition/available"; protected static final String VERSION_DEF_DEFINITION_BASE64 = PropertyHelper.getPropertyId(VERSION_DEF, VERSION_DEF_BASE64_PROPERTY); protected static final String VERSION_DEF_FULL_VERSION = "VersionDefinition/repository_version"; @@ -376,6 +376,8 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc String id = (String) propertyMap.get(VERSION_DEF_ID); if (null != id) { + // id is either the repo version id from the db, or it's a phantom id from + // the stack if (NumberUtils.isDigits(id)) { RepositoryVersionEntity entity = s_repoVersionDAO.findByPK(Long.parseLong(id)); http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index ddc7c60..f09c710 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -33,13 +33,12 @@ import org.apache.ambari.server.controller.StackVersionResponse; import org.apache.ambari.server.stack.Validable; import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.ConfigUpgradePack; +import org.apache.ambari.server.state.stack.LatestRepoCallable; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; import org.apache.ambari.server.state.stack.UpgradePack; -import com.google.common.collect.Collections2; import com.google.common.collect.ListMultimap; -import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; import com.google.common.io.Files; @@ -75,13 +74,15 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ private String upgradesFolder = null; private volatile Map<String, PropertyInfo> requiredProperties; private Map<String, VersionDefinitionXml> versionDefinitions = new ConcurrentHashMap<>(); - private Set<String> errorSet = new HashSet<String>(); + private Set<String> errorSet = new HashSet<>(); private RepositoryXml repoXml = null; + private VersionDefinitionXml latestVersion = null; + /** * List of services removed from current stack * */ - private List<String> removedServices = new ArrayList<String>(); + private List<String> removedServices = new ArrayList<>(); public String getMinJdk() { return minJdk; @@ -149,7 +150,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } public List<RepositoryInfo> getRepositories() { - if( repositories == null ) repositories = new ArrayList<RepositoryInfo>(); + if( repositories == null ) repositories = new ArrayList<>(); return repositories; } @@ -161,7 +162,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } public synchronized Collection<ServiceInfo> getServices() { - if (services == null) services = new ArrayList<ServiceInfo>(); + if (services == null) services = new ArrayList<>(); return services; } @@ -181,7 +182,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } public synchronized Collection<ExtensionInfo> getExtensions() { - if (extensions == null) extensions = new ArrayList<ExtensionInfo>(); + if (extensions == null) extensions = new ArrayList<>(); return extensions; } @@ -209,7 +210,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } public List<PropertyInfo> getProperties() { - if (properties == null) properties = new ArrayList<PropertyInfo>(); + if (properties == null) properties = new ArrayList<>(); return properties; } @@ -237,7 +238,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ */ public synchronized void setConfigTypeAttributes(String type, Map<String, Map<String, String>> typeAttributes) { if (this.configTypes == null) { - configTypes = new HashMap<String, Map<String, Map<String, String>>>(); + configTypes = new HashMap<>(); } // todo: no exclusion mechanism for stack config types configTypes.put(type, typeAttributes); @@ -250,7 +251,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ * @param types map of type attributes */ public synchronized void setAllConfigAttributes(Map<String, Map<String, Map<String, String>>> types) { - configTypes = new HashMap<String, Map<String, Map<String, String>>>(); + configTypes = new HashMap<>(); for (Map.Entry<String, Map<String, Map<String, String>>> entry : types.entrySet()) { setConfigTypeAttributes(entry.getKey(), entry.getValue()); } @@ -307,7 +308,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ // The collection of service descriptor files. A Set is being used because some Kerberos descriptor // files contain multiple services, therefore the same File may be encountered more than once. // For example the YARN directory may contain YARN and MAPREDUCE2 services. - Collection<File> serviceDescriptorFiles = new HashSet<File>(); + Collection<File> serviceDescriptorFiles = new HashSet<>(); if (serviceInfos != null) { for (ServiceInfo serviceInfo : serviceInfos) { File file = serviceInfo.getKerberosDescriptorFile(); @@ -468,7 +469,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ synchronized(this) { result = requiredProperties; if (result == null) { - requiredProperties = result = new HashMap<String, PropertyInfo>(); + requiredProperties = result = new HashMap<>(); List<PropertyInfo> properties = getProperties(); for (PropertyInfo propertyInfo : properties) { if (propertyInfo.isRequireInput()) { @@ -575,4 +576,18 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ public void setRemovedServices(List<String> removedServices) { this.removedServices = removedServices; } + + /** + * @param xml the version definition parsed from {@link LatestRepoCallable} + */ + public void setLatestVersionDefinition(VersionDefinitionXml xml) { + latestVersion = xml; + } + + /** + * @param xml the version definition parsed from {@link LatestRepoCallable} + */ + public VersionDefinitionXml getLatestVersionDefinition() { + return latestVersion; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java index 45d8e8e..de3b293 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java @@ -167,6 +167,13 @@ public class VersionDefinitionXml { } } + /** + * Sets if the version definition is a stack default. This can only be true + * when parsing "latest-vdf" for a stack. + */ + public void setStackDefault(boolean stackDefault) { + m_stackDefault = stackDefault; + } /** * Gets if the version definition was built as the default for a stack http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java index 3c7c001..c43ce7c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java @@ -22,6 +22,7 @@ import java.io.FileReader; import java.io.InputStreamReader; import java.lang.reflect.Type; import java.net.URI; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; @@ -46,7 +47,7 @@ import com.google.gson.reflect.TypeToken; */ public class LatestRepoCallable implements Callable<Void> { private static final int LOOKUP_CONNECTION_TIMEOUT = 2000; - private static final int LOOKUP_READ_TIMEOUT = 1000; + private static final int LOOKUP_READ_TIMEOUT = 3000; private final static Logger LOG = LoggerFactory.getLogger(LatestRepoCallable.class); @@ -70,6 +71,7 @@ public class LatestRepoCallable implements Callable<Void> { Map<String, Map<String, Object>> latestUrlMap = null; + Long time = System.currentTimeMillis(); try { if (sourceUri.startsWith("http")) { @@ -79,6 +81,7 @@ public class LatestRepoCallable implements Callable<Void> { LOG.info("Loading latest URL info for stack {}-{} from {}", stack.getName(), stack.getVersion(), sourceUri); + latestUrlMap = gson.fromJson(new InputStreamReader( streamProvider.readFrom(sourceUri)), type); } else { @@ -90,7 +93,7 @@ public class LatestRepoCallable implements Callable<Void> { } if (jsonFile.exists()) { - LOG.info("Loading latest URL info for stack{}-{} from {}", stack.getName(), + LOG.info("Loading latest URL info for stack {}-{} from {}", stack.getName(), stack.getVersion(), jsonFile); latestUrlMap = gson.fromJson(new FileReader(jsonFile), type); } @@ -99,6 +102,8 @@ public class LatestRepoCallable implements Callable<Void> { LOG.info("Could not load the URI for stack {}-{} from {}, ({}). Using default repository values", stack.getName(), stack.getVersion(), sourceUri, e.getMessage()); throw e; + } finally { + LOG.info("Loaded uri {} in {}ms", sourceUri, System.currentTimeMillis() - time); } // !!! process latest overrides @@ -154,32 +159,50 @@ public class LatestRepoCallable implements Callable<Void> { } Map<String, Object> map = latestUrlMap.get(stackId.toString()); - if (null == map || !map.containsKey("manifests")) { + if (null == map) { return null; } - @SuppressWarnings("unchecked") - Map<String, Object> versionMap = (Map<String, Object>) map.get("manifests"); + // !!! use this to prevent double loading of VDF. + Map<URI, VersionDefinitionXml> parsedMap = new HashMap<>(); - // EACH VDF is for ONLY ONE repository. We must provide a merged view. - // there is no good way around this, so we have to make some concessions + if (map.containsKey("manifests")) { - // !!! each key is a version number, and the value is a map containing - // os_family -> VDF link + @SuppressWarnings("unchecked") + Map<String, Object> versionMap = (Map<String, Object>) map.get("manifests"); - for (Entry<String, Object> entry : versionMap.entrySet()) { - String version = entry.getKey(); + // EACH VDF is for ONLY ONE repository. We must provide a merged view. + // there is no good way around this, so we have to make some concessions - @SuppressWarnings("unchecked") - Map<String, String> osMap = (Map<String, String>) entry.getValue(); + // !!! each key is a version number, and the value is a map containing + // os_family -> VDF link - VersionDefinitionXml xml = mergeDefinitions(stackId, version, osMap); + for (Entry<String, Object> entry : versionMap.entrySet()) { + String version = entry.getKey(); - if (null != xml) { - stack.addVersionDefinition(version, xml); + @SuppressWarnings("unchecked") + Map<String, String> osMap = (Map<String, String>) entry.getValue(); + + VersionDefinitionXml xml = mergeDefinitions(stackId, version, osMap, parsedMap); + + if (null != xml) { + stack.addVersionDefinition(version, xml); + } } } + if (map.containsKey("latest-vdf")) { + // !!! this is an os_family -> VDF link. It's identical to the version entry in the 'manifests' + // structure. This is the source of truth for the default, unspecified version when + // doing a bluerpint install + @SuppressWarnings("unchecked") + Map<String, String> osMap = (Map<String, String>) map.get("latest-vdf"); + + VersionDefinitionXml xml = mergeDefinitions(stackId, null, osMap, parsedMap); + xml.setStackDefault(true); + stack.setLatestVersionDefinition(xml); + } + return null; } @@ -191,7 +214,8 @@ public class LatestRepoCallable implements Callable<Void> { * @return the merged version definition * @throws Exception */ - private VersionDefinitionXml mergeDefinitions(StackId stackId, String version, Map<String, String> osMap) throws Exception { + private VersionDefinitionXml mergeDefinitions(StackId stackId, String version, + Map<String, String> osMap, Map<URI, VersionDefinitionXml> parsedMap) throws Exception { Set<String> oses = new HashSet<>(); for (RepositoryInfo ri : stack.getRepositories()) { @@ -230,9 +254,16 @@ public class LatestRepoCallable implements Callable<Void> { try { URI uri = new URI(uriString); - VersionDefinitionXml xml = VersionDefinitionXml.load(uri.toURL()); + VersionDefinitionXml xml = parsedMap.containsKey(uri) ? parsedMap.get(uri) : + timedVDFLoad(uri); + version = (null == version) ? xml.release.version : version; merger.add(version, xml); + + if (!parsedMap.containsKey(uri)) { + parsedMap.put(uri, xml); + } + } catch (Exception e) { LOG.warn("Could not load version definition for {} identified by {}. {}", stackId, uriString, e.getMessage(), e); @@ -242,8 +273,6 @@ public class LatestRepoCallable implements Callable<Void> { return merger.merge(); } - - /** * Resolves a base url given that certain OS types can be used interchangeably. * @param os the target os to find @@ -267,4 +296,15 @@ public class LatestRepoCallable implements Callable<Void> { return null; } + private VersionDefinitionXml timedVDFLoad(URI uri) throws Exception { + long time = System.currentTimeMillis(); + + try { + return VersionDefinitionXml.load(uri.toURL()); + } finally { + LOG.info("Loaded VDF {} in {}ms", uri, System.currentTimeMillis() - time); + } + + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index d77d13c..e8be965 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -61,9 +61,12 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.RequestImpl; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; import org.apache.ambari.server.controller.predicate.EqualsPredicate; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; @@ -129,6 +132,7 @@ public class AmbariContext { private static ServiceResourceProvider serviceResourceProvider; private static ComponentResourceProvider componentResourceProvider; private static HostComponentResourceProvider hostComponentResourceProvider; + private static VersionDefinitionResourceProvider versionDefinitionResourceProvider; private final static Logger LOG = LoggerFactory.getLogger(AmbariContext.class); @@ -204,7 +208,39 @@ public class AmbariContext { List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId); if (stackRepoVersions.isEmpty()) { - throw new IllegalArgumentException(String.format("No repositories were found for %s", stackId)); + // !!! no repos, try to get the version for the stack + VersionDefinitionResourceProvider vdfProvider = getVersionDefinitionResourceProvider(); + + Map<String, Object> properties = new HashMap<>(); + properties.put(VersionDefinitionResourceProvider.VERSION_DEF_AVAILABLE_DEFINITION, stackId.toString()); + + Request request = new RequestImpl(Collections.<String>emptySet(), + Collections.singleton(properties), Collections.<String, String>emptyMap(), null); + + Long repoVersionId = null; + + try { + RequestStatus requestStatus = vdfProvider.createResources(request); + if (!requestStatus.getAssociatedResources().isEmpty()) { + Resource resource = requestStatus.getAssociatedResources().iterator().next(); + repoVersionId = (Long) resource.getPropertyValue(VersionDefinitionResourceProvider.VERSION_DEF_ID); + } + } catch (Exception e) { + throw new IllegalArgumentException(String.format( + "Failed to create a default repository version definition for stack %s. " + + "This typically is a result of not loading the stack correctly or being able " + + "to load information about released versions. Create a repository version " + + " and try again.", stackId), e); + } + + repoVersion = repositoryVersionDAO.findByPK(repoVersionId); + // !!! better not! + if (null == repoVersion) { + throw new IllegalArgumentException(String.format( + "Failed to load the default repository version definition for stack %s. " + + "Check for a valid repository version and try again.", stackId)); + } + } else if (stackRepoVersions.size() > 1) { Function<RepositoryVersionEntity, String> function = new Function<RepositoryVersionEntity, String>() { @@ -786,4 +822,14 @@ public class AmbariContext { } return componentResourceProvider; } + + private synchronized VersionDefinitionResourceProvider getVersionDefinitionResourceProvider() { + if (versionDefinitionResourceProvider == null) { + versionDefinitionResourceProvider = (VersionDefinitionResourceProvider) ClusterControllerHelper. + getClusterController().ensureResourceProvider(Resource.Type.VersionDefinition); + } + return versionDefinitionResourceProvider; + + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 63cb21f..4c86476 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -84,6 +84,7 @@ import org.apache.ambari.server.state.alert.SourceType; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.Metric; import org.apache.ambari.server.state.stack.MetricDefinition; import org.apache.ambari.server.state.stack.OsFamily; @@ -1512,6 +1513,35 @@ public class AmbariMetaInfoTest { } } + + @Test + public void testLatestVdf() throws Exception { + // ensure that all of the latest repo retrieval tasks have completed + StackManager sm = metaInfo.getStackManager(); + int maxWait = 45000; + int waitTime = 0; + while (waitTime < maxWait && ! sm.haveAllRepoUrlsBeenResolved()) { + Thread.sleep(5); + waitTime += 5; + } + + if (waitTime >= maxWait) { + fail("Latest Repo tasks did not complete"); + } + + // !!! default stack version is from latest-vdf. 2.2.0 only has one entry + VersionDefinitionXml vdf = metaInfo.getVersionDefinition("HDP-2.2.0"); + assertNotNull(vdf); + assertEquals(1, vdf.repositoryInfo.getOses().size()); + + // !!! this stack has no "manifests" and no "latest-vdf". So the default VDF should contain + // information from repoinfo.xml and the "latest" structure + vdf = metaInfo.getVersionDefinition("HDP-2.2.1"); + assertNotNull(vdf); + + assertEquals(2, vdf.repositoryInfo.getOses().size()); + } + @Test public void testGetComponentDependency() throws AmbariException { DependencyInfo dependency = metaInfo.getComponentDependency("HDP", "1.3.4", "HIVE", "HIVE_SERVER", "ZOOKEEPER_SERVER"); http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 0809d63..5214f04 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -192,7 +192,7 @@ public class AmbariManagementControllerTest { private static final String PROPERTY_NAME = "hbase.regionserver.msginterval"; private static final String SERVICE_NAME = "HDFS"; private static final String FAKE_SERVICE_NAME = "FAKENAGIOS"; - private static final int STACK_VERSIONS_CNT = 16; + private static final int STACK_VERSIONS_CNT = 17; private static final int REPOS_CNT = 3; private static final int STACK_PROPERTIES_CNT = 103; private static final int STACK_COMPONENTS_CNT = 4; http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java index cb25aec..2c94747 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java @@ -60,7 +60,6 @@ import org.junit.Test; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; -import com.google.common.collect.Sets; import com.google.inject.Guice; import com.google.inject.Injector; @@ -277,7 +276,7 @@ public class VersionDefinitionResourceProviderTest { VersionDefinitionResourceProvider.SHOW_AVAILABLE).equals("true").toPredicate(); Set<Resource> results = versionProvider.getResources(getRequest, predicate); - Assert.assertEquals(2, results.size()); + Assert.assertEquals(3, results.size()); boolean found1 = false; boolean found2 = false; @@ -291,17 +290,10 @@ public class VersionDefinitionResourceProviderTest { VersionDefinitionXml vdf = ami.getVersionDefinition("HDP-2.2.0"); Assert.assertNotNull(vdf); - Assert.assertEquals(2, vdf.repositoryInfo.getOses().size()); + Assert.assertEquals(1, vdf.repositoryInfo.getOses().size()); String family1 = vdf.repositoryInfo.getOses().get(0).getFamily(); - String family2 = vdf.repositoryInfo.getOses().get(1).getFamily(); - - Assert.assertFalse(family1.equals(family2)); - Assert.assertTrue(Sets.newHashSet("suse11", "redhat6").contains(family1)); - Assert.assertTrue(Sets.newHashSet("suse11", "redhat6").contains(family2)); - - - + Assert.assertEquals("redhat6", family1); found2 = true; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index c9d3d2b..090bf55 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -131,13 +131,13 @@ public class StackManagerTest { @Test public void testGetsStacks() throws Exception { Collection<StackInfo> stacks = stackManager.getStacks(); - assertEquals(20, stacks.size()); + assertEquals(21, stacks.size()); } @Test public void testGetStacksByName() { Collection<StackInfo> stacks = stackManager.getStacks("HDP"); - assertEquals(16, stacks.size()); + assertEquals(17, stacks.size()); stacks = stackManager.getStacks("OTHER"); assertEquals(2, stacks.size()); @@ -164,7 +164,7 @@ public class StackManagerTest { List<String> removedServices = stack.getRemovedServices(); assertEquals(removedServices.size(), 2); - HashSet<String> expectedServices = new HashSet<String>(); + HashSet<String> expectedServices = new HashSet<>(); expectedServices.add("SPARK"); expectedServices.add("SPARK2"); @@ -186,7 +186,7 @@ public class StackManagerTest { Collection<ServiceInfo> services = stack.getServices(); assertEquals(3, services.size()); - Map<String, ServiceInfo> serviceMap = new HashMap<String, ServiceInfo>(); + Map<String, ServiceInfo> serviceMap = new HashMap<>(); for (ServiceInfo service : services) { serviceMap.put(service.getName(), service); } @@ -272,7 +272,7 @@ public class StackManagerTest { //should include all stacks in hierarchy assertEquals(17, services.size()); - HashSet<String> expectedServices = new HashSet<String>(); + HashSet<String> expectedServices = new HashSet<>(); expectedServices.add("GANGLIA"); expectedServices.add("HBASE"); expectedServices.add("HCATALOG"); @@ -383,7 +383,7 @@ public class StackManagerTest { // compare components List<ComponentInfo> stormServiceComponents = stormService.getComponents(); List<ComponentInfo> baseStormServiceComponents = baseStormService.getComponents(); - assertEquals(new HashSet<ComponentInfo>(stormServiceComponents), new HashSet<ComponentInfo>(baseStormServiceComponents)); + assertEquals(new HashSet<>(stormServiceComponents), new HashSet<>(baseStormServiceComponents)); // values from base service assertEquals(baseStormService.isDeleted(), stormService.isDeleted()); //todo: specify alerts file in stack @@ -516,7 +516,7 @@ public class StackManagerTest { Collection<ServiceInfo> allServices = stack.getServices(); assertEquals(12, allServices.size()); - HashSet<String> expectedServices = new HashSet<String>(); + HashSet<String> expectedServices = new HashSet<>(); expectedServices.add("GANGLIA"); expectedServices.add("HBASE"); expectedServices.add("HCATALOG"); http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index fc4fa86..fc53389 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -55,10 +55,12 @@ import org.apache.ambari.server.controller.internal.HostComponentResourceProvide import org.apache.ambari.server.controller.internal.HostResourceProvider; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; import org.apache.ambari.server.controller.predicate.EqualsPredicate; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; @@ -696,6 +698,20 @@ public class AmbariContextTest { @Test public void testCreateAmbariResourcesNoVersions() throws Exception { + VersionDefinitionResourceProvider vdfResourceProvider = createNiceMock(VersionDefinitionResourceProvider.class); + Class<AmbariContext> clazz = AmbariContext.class; + Field f = clazz.getDeclaredField("versionDefinitionResourceProvider"); + f.setAccessible(true); + f.set(null, vdfResourceProvider); + + Resource resource = createNiceMock(Resource.class); + expect(resource.getPropertyValue(VersionDefinitionResourceProvider.VERSION_DEF_ID)).andReturn(1L).atLeastOnce(); + + RequestStatus requestStatus = createNiceMock(RequestStatus.class); + expect(requestStatus.getAssociatedResources()).andReturn(Collections.singleton(resource)).atLeastOnce(); + + expect(vdfResourceProvider.createResources(EasyMock.anyObject(Request.class))).andReturn(requestStatus); + RepositoryVersionDAO repositoryVersionDAO = createNiceMock(RepositoryVersionDAO.class); RepositoryVersionEntity repositoryVersion = createNiceMock(RepositoryVersionEntity.class); expect(repositoryVersion.getId()).andReturn(1L).atLeastOnce(); @@ -703,19 +719,28 @@ public class AmbariContextTest { expect(repositoryVersionDAO.findByStack(EasyMock.anyObject(StackId.class))).andReturn( Collections.<RepositoryVersionEntity>emptyList()).atLeastOnce(); - replay(repositoryVersionDAO, repositoryVersion); + expect(repositoryVersionDAO.findByPK(EasyMock.anyLong())).andReturn(repositoryVersion); + + replay(repositoryVersionDAO, repositoryVersion, resource, requestStatus, vdfResourceProvider); context.repositoryVersionDAO = repositoryVersionDAO; + controller.createCluster(capture(Capture.<ClusterRequest>newInstance())); + expectLastCall().once(); + expect(cluster.getServices()).andReturn(clusterServices).anyTimes(); + + serviceResourceProvider.createServices(capture(Capture.<Set<ServiceRequest>>newInstance())); + expectLastCall().once(); + componentResourceProvider.createComponents(capture(Capture.<Set<ServiceComponentRequest>>newInstance())); + expectLastCall().once(); + + expect(serviceResourceProvider.updateResources( + capture(Capture.<Request>newInstance()), capture(Capture.<Predicate>newInstance()))).andReturn(null).atLeastOnce(); + replayAll(); // test - try { - context.createAmbariResources(topology, CLUSTER_NAME, null, null); - fail("Expected failure when no versions are found"); - } catch (IllegalArgumentException e) { - assertEquals("No repositories were found for testStack-testVersion", e.getMessage()); - } + context.createAmbariResources(topology, CLUSTER_NAME, null, null); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json index 708d757..fed67c1 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json @@ -10,8 +10,12 @@ "2.2.1.0": { "centos6": "./version-2.2.0.4-123.xml", "debian6": "./version-2.2.0.4-123.xml", + "ubuntu14": "./version-2.2.0.4-123.xml", "suse11sp3": "./version-2.2.0.4-123-suse11.xml" } + }, + "latest-vdf": { + "centos6": "./version-2.2.0.5.xml" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml index 031f5fd..b8bf342 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml @@ -33,4 +33,12 @@ <unique>true</unique> </repo> </os> + <os family="ubuntu14"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.2.0.0</baseurl> + <repoid>HDP-2.2.0</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + </os> </reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml new file mode 100644 index 0000000..8c1c982 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="version_definition.xsd"> + <release> + <type>STANDARD</type> + <stack-id>HDP-2.2.0</stack-id> + <version>2.2.0.5</version> + <build>123</build> + <release-notes>http://example.com</release-notes> + <display>HDP-2.2.0.4-1234</display> + <compatible-with>2.2.0.0</compatible-with> + </release> + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1.2.4"/> + <service id="HBASE-132" name="HBASE" version="1.3.2.4.3"/> + </manifest> + <available-services> + <service idref="HDFS-271"/> + </available-services> + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://baseurl1-2205</baseurl> + <repoid>HDP-2.4</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://baseurl2-2205</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + </repo> + </os> + </repository-info> +</repository-version> http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml new file mode 100644 index 0000000..43bcda5 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <active>true</active> + </versions> + <extends>2.2.0</extends> +</metainfo> + http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json new file mode 100644 index 0000000..88cdc2a --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json @@ -0,0 +1,7 @@ +{ + "HDP-2.1.1": { + "latest": { + "redhat6": "http://link/centos6/2.x/BUILDS/2.2.1.0-9999"; + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml new file mode 100644 index 0000000..5877d96 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml @@ -0,0 +1,36 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <latest>./hdp.json</latest> + <os family="redhat6"> + <repo> + <baseurl>http://link/centos6/2.x/updates/2.5.1.1</baseurl> + <repoid>HDP-2.5.0</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + </os> + <os family="ubuntu14"> + <repo> + <baseurl>http://link/ubuntu14/2.x/updates/2.5.0.0</baseurl> + <repoid>HDP-2.5.0</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/98be9abf/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json b/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json new file mode 100644 index 0000000..182a06c --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json @@ -0,0 +1,74 @@ +{ + "RANGER": { + "service": [], + "RANGER_ADMIN": [ + { + "name": "ranger_admin_process", + "label": "Ranger Admin Process", + "description": "This host-level alert is triggered if the Ranger Admin Web UI is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{admin-properties/policymgr_external_url}}", + "https": "{{admin-properties/policymgr_external_url}}", + "https_property": "{{ranger-admin-site/ranger.service.https.attrib.ssl.enabled}}", + "https_property_value": "true", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning": { + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + }, + { + "name": "ranger_admin_password_check", + "label": "Ranger Admin password check", + "description": "This alert is used to ensure that the Ranger Admin password in Ambari is correct.", + "interval": 30, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "RANGER/0.4.0/package/alerts/alert_ranger_admin_passwd_check.py", + "parameters": [] + } + } + ], + "RANGER_USERSYNC": [ + { + "name": "ranger_usersync_process", + "label": "Ranger Usersync Process", + "description": "This host-level alert is triggered if the Ranger Usersync cannot be determined to be up.", + "interval": 1, + "scope": "HOST", + "source": { + "type": "PORT", + "uri": "{{ranger-ugsync-site/ranger.usersync.port}}", + "default_port": 5151, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} \ No newline at end of file
