AMBARI-21756. Use latest-vdf for default when version is unspecified (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/15aa0375 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/15aa0375 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/15aa0375 Branch: refs/heads/branch-feature-logsearch-ui Commit: 15aa0375e8c79bbaab237a28c1f0b78275f551f6 Parents: 28a4223 Author: Nate Cole <nc...@hortonworks.com> Authored: Mon Aug 21 13:01:34 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Mon Aug 21 13:01:34 2017 -0400 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 9 ++- .../VersionDefinitionResourceProvider.java | 6 +- .../apache/ambari/server/state/StackInfo.java | 17 +++++ .../state/repository/VersionDefinitionXml.java | 7 ++ .../server/state/stack/LatestRepoCallable.java | 80 +++++++++++++++----- .../ambari/server/topology/AmbariContext.java | 50 +++++++++++- .../server/api/services/AmbariMetaInfoTest.java | 30 ++++++++ .../AmbariManagementControllerTest.java | 2 +- .../VersionDefinitionResourceProviderTest.java | 14 +--- .../ambari/server/stack/StackManagerTest.java | 5 +- .../server/topology/AmbariContextTest.java | 40 ++++++++-- .../resources/stacks/HDP/2.2.0/repos/hdp.json | 4 + .../stacks/HDP/2.2.0/repos/repoinfo.xml | 8 ++ .../stacks/HDP/2.2.0/repos/version-2.2.0.5.xml | 51 +++++++++++++ .../resources/stacks/HDP/2.2.1/metainfo.xml | 24 ++++++ .../resources/stacks/HDP/2.2.1/repos/hdp.json | 7 ++ .../stacks/HDP/2.2.1/repos/repoinfo.xml | 36 +++++++++ .../HDP/2.2.1/services/RANGER/alerts.json | 74 ++++++++++++++++++ 18 files changed, 418 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index eb97ee3..7eb82e4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -1381,7 +1381,14 @@ public class AmbariMetaInfo { if (stack.isActive() && stack.isValid()) { try { - VersionDefinitionXml xml = VersionDefinitionXml.build(stack); + // !!! check for a "latest-vdf" one. This will be used for the default if one is not found. + VersionDefinitionXml xml = stack.getLatestVersionDefinition(); + + if (null == xml) { + // !!! "latest-vdf" was not found, use the stack. this is the last-ditch effort + xml = VersionDefinitionXml.build(stack); + } + versionDefinitions.put(String.format("%s-%s", stack.getName(), stack.getVersion()), xml); } catch (Exception e) { LOG.warn("Could not make a stack VDF for {}-{}: {}", http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java index c789096..162c8df 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java @@ -91,10 +91,10 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc public static final String VERSION_DEF_STACK_NAME = "VersionDefinition/stack_name"; public static final String VERSION_DEF_STACK_VERSION = "VersionDefinition/stack_version"; - protected static final String VERSION_DEF_ID = "VersionDefinition/id"; + public static final String VERSION_DEF_ID = "VersionDefinition/id"; protected static final String VERSION_DEF_TYPE_PROPERTY_ID = "VersionDefinition/type"; protected static final String VERSION_DEF_DEFINITION_URL = "VersionDefinition/version_url"; - protected static final String VERSION_DEF_AVAILABLE_DEFINITION = "VersionDefinition/available"; + public static final String VERSION_DEF_AVAILABLE_DEFINITION = "VersionDefinition/available"; protected static final String VERSION_DEF_DEFINITION_BASE64 = PropertyHelper.getPropertyId(VERSION_DEF, VERSION_DEF_BASE64_PROPERTY); protected static final String VERSION_DEF_FULL_VERSION = "VersionDefinition/repository_version"; @@ -378,6 +378,8 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc String id = (String) propertyMap.get(VERSION_DEF_ID); if (null != id) { + // id is either the repo version id from the db, or it's a phantom id from + // the stack if (NumberUtils.isDigits(id)) { RepositoryVersionEntity entity = s_repoVersionDAO.findByPK(Long.parseLong(id)); http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index 149579f..350611d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -33,6 +33,7 @@ import org.apache.ambari.server.controller.StackVersionResponse; import org.apache.ambari.server.stack.Validable; import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.ConfigUpgradePack; +import org.apache.ambari.server.state.stack.LatestRepoCallable; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; import org.apache.ambari.server.state.stack.UpgradePack; @@ -77,6 +78,8 @@ public class StackInfo implements Comparable<StackInfo>, Validable { private Set<String> errorSet = new HashSet<>(); private RepositoryXml repoXml = null; + private VersionDefinitionXml latestVersion = null; + /** * List of services removed from current stack * */ @@ -607,4 +610,18 @@ public class StackInfo implements Comparable<StackInfo>, Validable { public void setServicesWithNoConfigs(List<String> servicesWithNoConfigs) { this.servicesWithNoConfigs = servicesWithNoConfigs; } + + /** + * @param xml the version definition parsed from {@link LatestRepoCallable} + */ + public void setLatestVersionDefinition(VersionDefinitionXml xml) { + latestVersion = xml; + } + + /** + * @param xml the version definition parsed from {@link LatestRepoCallable} + */ + public VersionDefinitionXml getLatestVersionDefinition() { + return latestVersion; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java index 3ef3991..96bf668 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java @@ -167,6 +167,13 @@ public class VersionDefinitionXml { } } + /** + * Sets if the version definition is a stack default. This can only be true + * when parsing "latest-vdf" for a stack. + */ + public void setStackDefault(boolean stackDefault) { + m_stackDefault = stackDefault; + } /** * Gets if the version definition was built as the default for a stack http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java index 0472bc1..90d562f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java @@ -22,6 +22,7 @@ import java.io.FileReader; import java.io.InputStreamReader; import java.lang.reflect.Type; import java.net.URI; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; @@ -46,7 +47,7 @@ import com.google.gson.reflect.TypeToken; */ public class LatestRepoCallable implements Callable<Void> { private static final int LOOKUP_CONNECTION_TIMEOUT = 2000; - private static final int LOOKUP_READ_TIMEOUT = 1000; + private static final int LOOKUP_READ_TIMEOUT = 3000; private final static Logger LOG = LoggerFactory.getLogger(LatestRepoCallable.class); @@ -70,6 +71,7 @@ public class LatestRepoCallable implements Callable<Void> { Map<String, Map<String, Object>> latestUrlMap = null; + Long time = System.currentTimeMillis(); try { if (sourceUri.startsWith("http")) { @@ -79,6 +81,7 @@ public class LatestRepoCallable implements Callable<Void> { LOG.info("Loading latest URL info for stack {}-{} from {}", stack.getName(), stack.getVersion(), sourceUri); + latestUrlMap = gson.fromJson(new InputStreamReader( streamProvider.readFrom(sourceUri)), type); } else { @@ -90,7 +93,7 @@ public class LatestRepoCallable implements Callable<Void> { } if (jsonFile.exists()) { - LOG.info("Loading latest URL info for stack{}-{} from {}", stack.getName(), + LOG.info("Loading latest URL info for stack {}-{} from {}", stack.getName(), stack.getVersion(), jsonFile); latestUrlMap = gson.fromJson(new FileReader(jsonFile), type); } @@ -99,6 +102,8 @@ public class LatestRepoCallable implements Callable<Void> { LOG.info("Could not load the URI for stack {}-{} from {}, ({}). Using default repository values", stack.getName(), stack.getVersion(), sourceUri, e.getMessage()); throw e; + } finally { + LOG.info("Loaded uri {} in {}ms", sourceUri, System.currentTimeMillis() - time); } StackId stackId = new StackId(stack); @@ -107,32 +112,50 @@ public class LatestRepoCallable implements Callable<Void> { } Map<String, Object> map = latestUrlMap.get(stackId.toString()); - if (null == map || !map.containsKey("manifests")) { + if (null == map) { return null; } - @SuppressWarnings("unchecked") - Map<String, Object> versionMap = (Map<String, Object>) map.get("manifests"); + // !!! use this to prevent double loading of VDF. + Map<URI, VersionDefinitionXml> parsedMap = new HashMap<>(); - // EACH VDF is for ONLY ONE repository. We must provide a merged view. - // there is no good way around this, so we have to make some concessions + if (map.containsKey("manifests")) { - // !!! each key is a version number, and the value is a map containing - // os_family -> VDF link + @SuppressWarnings("unchecked") + Map<String, Object> versionMap = (Map<String, Object>) map.get("manifests"); - for (Entry<String, Object> entry : versionMap.entrySet()) { - String version = entry.getKey(); + // EACH VDF is for ONLY ONE repository. We must provide a merged view. + // there is no good way around this, so we have to make some concessions - @SuppressWarnings("unchecked") - Map<String, String> osMap = (Map<String, String>) entry.getValue(); + // !!! each key is a version number, and the value is a map containing + // os_family -> VDF link - VersionDefinitionXml xml = mergeDefinitions(stackId, version, osMap); + for (Entry<String, Object> entry : versionMap.entrySet()) { + String version = entry.getKey(); - if (null != xml) { - stack.addVersionDefinition(version, xml); + @SuppressWarnings("unchecked") + Map<String, String> osMap = (Map<String, String>) entry.getValue(); + + VersionDefinitionXml xml = mergeDefinitions(stackId, version, osMap, parsedMap); + + if (null != xml) { + stack.addVersionDefinition(version, xml); + } } } + if (map.containsKey("latest-vdf")) { + // !!! this is an os_family -> VDF link. It's identical to the version entry in the 'manifests' + // structure. This is the source of truth for the default, unspecified version when + // doing a bluerpint install + @SuppressWarnings("unchecked") + Map<String, String> osMap = (Map<String, String>) map.get("latest-vdf"); + + VersionDefinitionXml xml = mergeDefinitions(stackId, null, osMap, parsedMap); + xml.setStackDefault(true); + stack.setLatestVersionDefinition(xml); + } + return null; } @@ -144,7 +167,8 @@ public class LatestRepoCallable implements Callable<Void> { * @return the merged version definition * @throws Exception */ - private VersionDefinitionXml mergeDefinitions(StackId stackId, String version, Map<String, String> osMap) throws Exception { + private VersionDefinitionXml mergeDefinitions(StackId stackId, String version, + Map<String, String> osMap, Map<URI, VersionDefinitionXml> parsedMap) throws Exception { Set<String> oses = new HashSet<>(); for (RepositoryInfo ri : stack.getRepositories()) { @@ -183,9 +207,16 @@ public class LatestRepoCallable implements Callable<Void> { try { URI uri = new URI(uriString); - VersionDefinitionXml xml = VersionDefinitionXml.load(uri.toURL()); + VersionDefinitionXml xml = parsedMap.containsKey(uri) ? parsedMap.get(uri) : + timedVDFLoad(uri); + version = (null == version) ? xml.release.version : version; merger.add(version, xml); + + if (!parsedMap.containsKey(uri)) { + parsedMap.put(uri, xml); + } + } catch (Exception e) { LOG.warn("Could not load version definition for {} identified by {}. {}", stackId, uriString, e.getMessage(), e); @@ -195,8 +226,6 @@ public class LatestRepoCallable implements Callable<Void> { return merger.merge(); } - - /** * Resolves a base url given that certain OS types can be used interchangeably. * @param os the target os to find @@ -220,4 +249,15 @@ public class LatestRepoCallable implements Callable<Void> { return null; } + private VersionDefinitionXml timedVDFLoad(URI uri) throws Exception { + long time = System.currentTimeMillis(); + + try { + return VersionDefinitionXml.load(uri.toURL()); + } finally { + LOG.info("Loaded VDF {} in {}ms", uri, System.currentTimeMillis() - time); + } + + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index fa703ed..75435a3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -61,9 +61,12 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.RequestImpl; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; import org.apache.ambari.server.controller.predicate.EqualsPredicate; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; @@ -130,6 +133,7 @@ public class AmbariContext { private static ServiceResourceProvider serviceResourceProvider; private static ComponentResourceProvider componentResourceProvider; private static HostComponentResourceProvider hostComponentResourceProvider; + private static VersionDefinitionResourceProvider versionDefinitionResourceProvider; private final static Logger LOG = LoggerFactory.getLogger(AmbariContext.class); @@ -205,7 +209,39 @@ public class AmbariContext { List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId); if (stackRepoVersions.isEmpty()) { - throw new IllegalArgumentException(String.format("No repositories were found for %s", stackId)); + // !!! no repos, try to get the version for the stack + VersionDefinitionResourceProvider vdfProvider = getVersionDefinitionResourceProvider(); + + Map<String, Object> properties = new HashMap<>(); + properties.put(VersionDefinitionResourceProvider.VERSION_DEF_AVAILABLE_DEFINITION, stackId.toString()); + + Request request = new RequestImpl(Collections.<String>emptySet(), + Collections.singleton(properties), Collections.<String, String>emptyMap(), null); + + Long repoVersionId = null; + + try { + RequestStatus requestStatus = vdfProvider.createResources(request); + if (!requestStatus.getAssociatedResources().isEmpty()) { + Resource resource = requestStatus.getAssociatedResources().iterator().next(); + repoVersionId = (Long) resource.getPropertyValue(VersionDefinitionResourceProvider.VERSION_DEF_ID); + } + } catch (Exception e) { + throw new IllegalArgumentException(String.format( + "Failed to create a default repository version definition for stack %s. " + + "This typically is a result of not loading the stack correctly or being able " + + "to load information about released versions. Create a repository version " + + " and try again.", stackId), e); + } + + repoVersion = repositoryVersionDAO.findByPK(repoVersionId); + // !!! better not! + if (null == repoVersion) { + throw new IllegalArgumentException(String.format( + "Failed to load the default repository version definition for stack %s. " + + "Check for a valid repository version and try again.", stackId)); + } + } else if (stackRepoVersions.size() > 1) { Function<RepositoryVersionEntity, String> function = new Function<RepositoryVersionEntity, String>() { @@ -794,4 +830,14 @@ public class AmbariContext { } return componentResourceProvider; } -} + + private synchronized VersionDefinitionResourceProvider getVersionDefinitionResourceProvider() { + if (versionDefinitionResourceProvider == null) { + versionDefinitionResourceProvider = (VersionDefinitionResourceProvider) ClusterControllerHelper. + getClusterController().ensureResourceProvider(Resource.Type.VersionDefinition); + } + return versionDefinitionResourceProvider; + + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 5afe87e..9ea0ae1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -84,6 +84,7 @@ import org.apache.ambari.server.state.alert.SourceType; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.Metric; import org.apache.ambari.server.state.stack.MetricDefinition; import org.apache.ambari.server.state.stack.OsFamily; @@ -1519,6 +1520,35 @@ public class AmbariMetaInfoTest { } } + + @Test + public void testLatestVdf() throws Exception { + // ensure that all of the latest repo retrieval tasks have completed + StackManager sm = metaInfo.getStackManager(); + int maxWait = 45000; + int waitTime = 0; + while (waitTime < maxWait && ! sm.haveAllRepoUrlsBeenResolved()) { + Thread.sleep(5); + waitTime += 5; + } + + if (waitTime >= maxWait) { + fail("Latest Repo tasks did not complete"); + } + + // !!! default stack version is from latest-vdf. 2.2.0 only has one entry + VersionDefinitionXml vdf = metaInfo.getVersionDefinition("HDP-2.2.0"); + assertNotNull(vdf); + assertEquals(1, vdf.repositoryInfo.getOses().size()); + + // !!! this stack has no "manifests" and no "latest-vdf". So the default VDF should contain + // information from repoinfo.xml and the "latest" structure + vdf = metaInfo.getVersionDefinition("HDP-2.2.1"); + assertNotNull(vdf); + + assertEquals(2, vdf.repositoryInfo.getOses().size()); + } + @Test public void testGetComponentDependency() throws AmbariException { DependencyInfo dependency = metaInfo.getComponentDependency("HDP", "1.3.4", "HIVE", "HIVE_SERVER", "ZOOKEEPER_SERVER"); http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 85ad36d..9309abe 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -192,7 +192,7 @@ public class AmbariManagementControllerTest { private static final String PROPERTY_NAME = "hbase.regionserver.msginterval"; private static final String SERVICE_NAME = "HDFS"; private static final String FAKE_SERVICE_NAME = "FAKENAGIOS"; - private static final int STACK_VERSIONS_CNT = 16; + private static final int STACK_VERSIONS_CNT = 17; private static final int REPOS_CNT = 3; private static final int STACK_PROPERTIES_CNT = 103; private static final int STACK_COMPONENTS_CNT = 4; http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java index 7e1db0d..402a8f8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java @@ -60,7 +60,6 @@ import org.junit.Test; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; -import com.google.common.collect.Sets; import com.google.inject.Guice; import com.google.inject.Injector; @@ -277,7 +276,7 @@ public class VersionDefinitionResourceProviderTest { VersionDefinitionResourceProvider.SHOW_AVAILABLE).equals("true").toPredicate(); Set<Resource> results = versionProvider.getResources(getRequest, predicate); - Assert.assertEquals(2, results.size()); + Assert.assertEquals(3, results.size()); boolean found1 = false; boolean found2 = false; @@ -291,17 +290,10 @@ public class VersionDefinitionResourceProviderTest { VersionDefinitionXml vdf = ami.getVersionDefinition("HDP-2.2.0"); Assert.assertNotNull(vdf); - Assert.assertEquals(2, vdf.repositoryInfo.getOses().size()); + Assert.assertEquals(1, vdf.repositoryInfo.getOses().size()); String family1 = vdf.repositoryInfo.getOses().get(0).getFamily(); - String family2 = vdf.repositoryInfo.getOses().get(1).getFamily(); - - Assert.assertFalse(family1.equals(family2)); - Assert.assertTrue(Sets.newHashSet("suse11", "redhat6").contains(family1)); - Assert.assertTrue(Sets.newHashSet("suse11", "redhat6").contains(family2)); - - - + Assert.assertEquals("redhat6", family1); found2 = true; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index 869441f..f53e111 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -133,13 +133,13 @@ public class StackManagerTest { @Test public void testGetsStacks() throws Exception { Collection<StackInfo> stacks = stackManager.getStacks(); - assertEquals(20, stacks.size()); + assertEquals(21, stacks.size()); } @Test public void testGetStacksByName() { Collection<StackInfo> stacks = stackManager.getStacks("HDP"); - assertEquals(16, stacks.size()); + assertEquals(17, stacks.size()); stacks = stackManager.getStacks("OTHER"); assertEquals(2, stacks.size()); @@ -289,6 +289,7 @@ public class StackManagerTest { //should include all stacks in hierarchy assertEquals(18, services.size()); + HashSet<String> expectedServices = new HashSet<>(); expectedServices.add("GANGLIA"); expectedServices.add("HBASE"); http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index bef0a45..f8ed815 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -55,10 +55,12 @@ import org.apache.ambari.server.controller.internal.HostComponentResourceProvide import org.apache.ambari.server.controller.internal.HostResourceProvider; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; import org.apache.ambari.server.controller.predicate.EqualsPredicate; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; @@ -699,26 +701,50 @@ public class AmbariContextTest { @Test public void testCreateAmbariResourcesNoVersions() throws Exception { + VersionDefinitionResourceProvider vdfResourceProvider = createNiceMock(VersionDefinitionResourceProvider.class); + Class<AmbariContext> clazz = AmbariContext.class; + Field f = clazz.getDeclaredField("versionDefinitionResourceProvider"); + f.setAccessible(true); + f.set(null, vdfResourceProvider); + + Resource resource = createNiceMock(Resource.class); + expect(resource.getPropertyValue(VersionDefinitionResourceProvider.VERSION_DEF_ID)).andReturn(1L).atLeastOnce(); + + RequestStatus requestStatus = createNiceMock(RequestStatus.class); + expect(requestStatus.getAssociatedResources()).andReturn(Collections.singleton(resource)).atLeastOnce(); + + expect(vdfResourceProvider.createResources(EasyMock.anyObject(Request.class))).andReturn(requestStatus); + RepositoryVersionDAO repositoryVersionDAO = createNiceMock(RepositoryVersionDAO.class); RepositoryVersionEntity repositoryVersion = createNiceMock(RepositoryVersionEntity.class); expect(repositoryVersion.getId()).andReturn(1L).atLeastOnce(); expect(repositoryVersion.getVersion()).andReturn("1.1.1.1").atLeastOnce(); + expect(repositoryVersion.getType()).andReturn(RepositoryType.STANDARD).atLeastOnce(); expect(repositoryVersionDAO.findByStack(EasyMock.anyObject(StackId.class))).andReturn( Collections.<RepositoryVersionEntity>emptyList()).atLeastOnce(); - replay(repositoryVersionDAO, repositoryVersion); + expect(repositoryVersionDAO.findByPK(EasyMock.anyLong())).andReturn(repositoryVersion); + + replay(repositoryVersionDAO, repositoryVersion, resource, requestStatus, vdfResourceProvider); context.repositoryVersionDAO = repositoryVersionDAO; + controller.createCluster(capture(Capture.<ClusterRequest>newInstance())); + expectLastCall().once(); + expect(cluster.getServices()).andReturn(clusterServices).anyTimes(); + + serviceResourceProvider.createServices(capture(Capture.<Set<ServiceRequest>>newInstance())); + expectLastCall().once(); + componentResourceProvider.createComponents(capture(Capture.<Set<ServiceComponentRequest>>newInstance())); + expectLastCall().once(); + + expect(serviceResourceProvider.updateResources( + capture(Capture.<Request>newInstance()), capture(Capture.<Predicate>newInstance()))).andReturn(null).atLeastOnce(); + replayAll(); // test - try { - context.createAmbariResources(topology, CLUSTER_NAME, null, null); - fail("Expected failure when no versions are found"); - } catch (IllegalArgumentException e) { - assertEquals("No repositories were found for testStack-testVersion", e.getMessage()); - } + context.createAmbariResources(topology, CLUSTER_NAME, null, null); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json index 708d757..fed67c1 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json @@ -10,8 +10,12 @@ "2.2.1.0": { "centos6": "./version-2.2.0.4-123.xml", "debian6": "./version-2.2.0.4-123.xml", + "ubuntu14": "./version-2.2.0.4-123.xml", "suse11sp3": "./version-2.2.0.4-123-suse11.xml" } + }, + "latest-vdf": { + "centos6": "./version-2.2.0.5.xml" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml index 031f5fd..b8bf342 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/repoinfo.xml @@ -33,4 +33,12 @@ <unique>true</unique> </repo> </os> + <os family="ubuntu14"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.2.0.0</baseurl> + <repoid>HDP-2.2.0</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + </os> </reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml new file mode 100644 index 0000000..8c1c982 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.5.xml @@ -0,0 +1,51 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="version_definition.xsd"> + <release> + <type>STANDARD</type> + <stack-id>HDP-2.2.0</stack-id> + <version>2.2.0.5</version> + <build>123</build> + <release-notes>http://example.com</release-notes> + <display>HDP-2.2.0.4-1234</display> + <compatible-with>2.2.0.0</compatible-with> + </release> + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1.2.4"/> + <service id="HBASE-132" name="HBASE" version="1.3.2.4.3"/> + </manifest> + <available-services> + <service idref="HDFS-271"/> + </available-services> + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://baseurl1-2205</baseurl> + <repoid>HDP-2.4</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://baseurl2-2205</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + </repo> + </os> + </repository-info> +</repository-version> http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml new file mode 100644 index 0000000..43bcda5 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/metainfo.xml @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <active>true</active> + </versions> + <extends>2.2.0</extends> +</metainfo> + http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json new file mode 100644 index 0000000..88cdc2a --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/hdp.json @@ -0,0 +1,7 @@ +{ + "HDP-2.1.1": { + "latest": { + "redhat6": "http://link/centos6/2.x/BUILDS/2.2.1.0-9999"; + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml new file mode 100644 index 0000000..5877d96 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/repos/repoinfo.xml @@ -0,0 +1,36 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <latest>./hdp.json</latest> + <os family="redhat6"> + <repo> + <baseurl>http://link/centos6/2.x/updates/2.5.1.1</baseurl> + <repoid>HDP-2.5.0</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + </os> + <os family="ubuntu14"> + <repo> + <baseurl>http://link/ubuntu14/2.x/updates/2.5.0.0</baseurl> + <repoid>HDP-2.5.0</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/15aa0375/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json b/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json new file mode 100644 index 0000000..182a06c --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.1/services/RANGER/alerts.json @@ -0,0 +1,74 @@ +{ + "RANGER": { + "service": [], + "RANGER_ADMIN": [ + { + "name": "ranger_admin_process", + "label": "Ranger Admin Process", + "description": "This host-level alert is triggered if the Ranger Admin Web UI is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{admin-properties/policymgr_external_url}}", + "https": "{{admin-properties/policymgr_external_url}}", + "https_property": "{{ranger-admin-site/ranger.service.https.attrib.ssl.enabled}}", + "https_property_value": "true", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning": { + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + }, + { + "name": "ranger_admin_password_check", + "label": "Ranger Admin password check", + "description": "This alert is used to ensure that the Ranger Admin password in Ambari is correct.", + "interval": 30, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "RANGER/0.4.0/package/alerts/alert_ranger_admin_passwd_check.py", + "parameters": [] + } + } + ], + "RANGER_USERSYNC": [ + { + "name": "ranger_usersync_process", + "label": "Ranger Usersync Process", + "description": "This host-level alert is triggered if the Ranger Usersync cannot be determined to be up.", + "interval": 1, + "scope": "HOST", + "source": { + "type": "PORT", + "uri": "{{ranger-ugsync-site/ranger.usersync.port}}", + "default_port": 5151, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} \ No newline at end of file
