AMBARI-22461. VDF defined HDP-GPL repo should be tagged appropriately (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6cc05d95 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6cc05d95 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6cc05d95 Branch: refs/heads/branch-2.6 Commit: 6cc05d95c16b0ead495159aa58be3b221bd67530 Parents: 46237fe Author: Nate Cole <[email protected]> Authored: Thu Nov 16 08:42:20 2017 -0500 Committer: Nate Cole <[email protected]> Committed: Thu Nov 16 11:03:04 2017 -0500 ---------------------------------------------------------------------- .../AmbariManagementControllerImpl.java | 8 +- .../server/controller/RepositoryResponse.java | 23 +++++- .../internal/RepositoryResourceProvider.java | 11 ++- .../VersionDefinitionResourceProvider.java | 8 ++ .../server/orm/entities/RepositoryEntity.java | 20 +++++ .../apache/ambari/server/stack/RepoUtil.java | 1 + .../ambari/server/state/RepositoryInfo.java | 24 +++++- .../ambari/server/state/stack/RepoTag.java | 34 ++++++++ .../server/state/stack/RepositoryXml.java | 17 +++- .../stack/upgrade/RepositoryVersionHelper.java | 19 +++++ .../src/main/resources/version_definition.xsd | 23 +++++- .../RepositoryResourceProviderTest.java | 15 ++-- .../state/repository/VersionDefinitionTest.java | 35 +++++++- .../upgrade/RepositoryVersionHelperTest.java | 2 +- .../resources/version_definition_with_tags.xml | 86 ++++++++++++++++++++ contrib/version-builder/version_builder.py | 13 ++- 16 files changed, 314 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index a6315f8..ffa8da3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -88,7 +88,6 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.RequestFactory; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; -import org.apache.ambari.server.agent.CommandRepository; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; @@ -193,6 +192,7 @@ import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfile; import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.scheduler.RequestExecutionFactory; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.state.stack.RepoTag; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.WidgetLayout; import org.apache.ambari.server.state.stack.WidgetLayoutInfo; @@ -4469,7 +4469,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle final RepositoryResponse response = new RepositoryResponse(repository.getBaseUrl(), osType, repository.getRepositoryId(), repository.getName(), repository.getDistribution(), repository.getComponents(), "", "", "", - repository.getApplicableServices()); + repository.getApplicableServices(), + repository.getTags()); if (null != versionDefinitionId) { response.setVersionDefinitionId(versionDefinitionId); } else { @@ -4477,6 +4478,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } response.setStackName(repositoryVersion.getStackName()); response.setStackVersion(repositoryVersion.getStackVersion()); + responses.add(response); } break; @@ -4498,7 +4500,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle for (RepositoryXml.Repo repo : os.getRepos()) { RepositoryResponse resp = new RepositoryResponse(repo.getBaseUrl(), os.getFamily(), repo.getRepoId(), repo.getRepoName(), repo.getDistribution(), repo.getComponents(), repo.getMirrorsList(), - repo.getBaseUrl(), repo.getLatestUri(), Collections.EMPTY_LIST); + repo.getBaseUrl(), repo.getLatestUri(), Collections.<String>emptyList(), Collections.<RepoTag>emptySet()); resp.setVersionDefinitionId(versionDefinitionId); resp.setStackName(stackId.getStackName()); http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java index 5ec46f0..46a82bb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java @@ -19,9 +19,11 @@ package org.apache.ambari.server.controller; import java.util.List; +import java.util.Set; import org.apache.ambari.annotations.Experimental; import org.apache.ambari.annotations.ExperimentalFeature; +import org.apache.ambari.server.state.stack.RepoTag; public class RepositoryResponse { @@ -45,9 +47,12 @@ public class RepositoryResponse { comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment") private List<String> applicableServices; + private Set<RepoTag> tags; + public RepositoryResponse(String baseUrl, String osType, String repoId, String repoName, String distribution, String components, - String mirrorsList, String defaultBaseUrl, String latestBaseUrl, List<String> applicableServices) { + String mirrorsList, String defaultBaseUrl, String latestBaseUrl, List<String> applicableServices, + Set<RepoTag> repoTags) { setBaseUrl(baseUrl); setOsType(osType); setRepoId(repoId); @@ -58,6 +63,7 @@ public class RepositoryResponse { setDefaultBaseUrl(defaultBaseUrl); setLatestBaseUrl(latestBaseUrl); setApplicableServices(applicableServices); + setTags(repoTags); } public String getStackName() { @@ -203,4 +209,19 @@ public class RepositoryResponse { public void setApplicableServices(List<String> applicableServices) { this.applicableServices = applicableServices; } + + /** + * @return the repo tags + */ + public Set<RepoTag> getTags() { + return tags; + } + + /** + * @param repoTags the repo tags + */ + public void setTags(Set<RepoTag> repoTags) { + tags = repoTags; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java index 3c10e43..cec43dd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java @@ -64,6 +64,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid public static final String REPOSITORY_REPOSITORY_VERSION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "repository_version_id"); public static final String REPOSITORY_VERSION_DEFINITION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "version_definition_id"); public static final String REPOSITORY_UNIQUE_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "unique"); + public static final String REPOSITORY_TAGS_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "tags"); @Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS, comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment") public static final String REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "applicable_services"); @@ -98,6 +99,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid add(REPOSITORY_CLUSTER_STACK_VERSION_PROPERTY_ID); add(REPOSITORY_UNIQUE_PROPERTY_ID); add(REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID); + add(REPOSITORY_TAGS_PROPERTY_ID); } }; @@ -123,7 +125,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { - final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>(); + final Set<RepositoryRequest> requests = new HashSet<>(); Iterator<Map<String,Object>> iterator = request.getProperties().iterator(); if (iterator.hasNext()) { @@ -148,7 +150,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { - final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>(); + final Set<RepositoryRequest> requests = new HashSet<>(); if (predicate == null) { requests.add(getRequest(Collections.<String, Object>emptyMap())); @@ -166,7 +168,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid } }); - Set<Resource> resources = new HashSet<Resource>(); + Set<Resource> resources = new HashSet<>(); for (RepositoryResponse response : responses) { Resource resource = new ResourceImpl(Resource.Type.Repository); @@ -184,6 +186,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid setResourceProperty(resource, REPOSITORY_LATEST_BASE_URL_PROPERTY_ID, response.getLatestBaseUrl(), requestedIds); setResourceProperty(resource, REPOSITORY_UNIQUE_PROPERTY_ID, response.isUnique(), requestedIds); setResourceProperty(resource, REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID, response.getApplicableServices(), requestedIds); + setResourceProperty(resource, REPOSITORY_TAGS_PROPERTY_ID, response.getTags(), requestedIds); if (null != response.getClusterVersionId()) { setResourceProperty(resource, REPOSITORY_CLUSTER_STACK_VERSION_PROPERTY_ID, response.getClusterVersionId(), requestedIds); } @@ -207,7 +210,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid public RequestStatus createResources(Request request) throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException { final String validateOnlyProperty = request.getRequestInfoProperties().get(RepositoryResourceDefinition.VALIDATE_ONLY_DIRECTIVE); if (BooleanUtils.toBoolean(validateOnlyProperty)) { - final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>(); + final Set<RepositoryRequest> requests = new HashSet<>(); final Iterator<Map<String,Object>> iterator = request.getProperties().iterator(); if (iterator.hasNext()) { for (Map<String, Object> propertyMap : request.getProperties()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java index 3ca81d7..6ab863b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java @@ -63,6 +63,7 @@ import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.repository.VersionDefinitionXml; +import org.apache.ambari.server.state.stack.RepoTag; import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.IOUtils; @@ -780,6 +781,13 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID), entity.getStackVersion()); + ArrayNode tagsNode = factory.arrayNode(); + for (RepoTag repoTag : repo.getTags()) { + tagsNode.add(repoTag.toString()); + } + repoElement.put(PropertyHelper.getPropertyName( + RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID), tagsNode); + @Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS, comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment") ArrayNode applicableServicesNode = factory.arrayNode(); http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java index 034f3bc..5f23ca7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java @@ -17,10 +17,13 @@ */ package org.apache.ambari.server.orm.entities; +import java.util.Collections; import java.util.List; +import java.util.Set; import org.apache.ambari.annotations.Experimental; import org.apache.ambari.annotations.ExperimentalFeature; +import org.apache.ambari.server.state.stack.RepoTag; /** * Emulates entity to provide a quick way to change it to real entity in future. @@ -38,6 +41,8 @@ public class RepositoryEntity { comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment") private List<String> applicableServices; + private Set<RepoTag> tags; + public String getName() { return name; } @@ -106,6 +111,20 @@ public class RepositoryEntity { this.applicableServices = applicableServices; } + /** + * @return the repo tags + */ + public Set<RepoTag> getTags() { + return tags == null ? Collections.<RepoTag>emptySet() : tags; + } + + /** + * @param repoTags the tags to set + */ + public void setTags(Set<RepoTag> repoTags) { + tags = repoTags; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -132,4 +151,5 @@ public class RepositoryEntity { result = 31 * result + (applicableServices != null ? applicableServices.hashCode() : 0); return result; } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java index 81b4da9..166abb5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java @@ -200,6 +200,7 @@ public class RepoUtil { re.setDistribution(repoInfo.getDistribution()); re.setComponents(repoInfo.getComponents()); re.setApplicableServices(repoInfo.getApplicableServices()); + re.setTags(repoInfo.getTags()); return re; } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java index bd53c11..9272d56 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java @@ -18,16 +18,19 @@ package org.apache.ambari.server.state; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import com.google.common.base.Objects; +import java.util.Set; import org.apache.ambari.annotations.Experimental; import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.controller.RepositoryResponse; +import org.apache.ambari.server.state.stack.RepoTag; import org.apache.commons.lang.StringUtils; import com.google.common.base.Function; +import com.google.common.base.Objects; import com.google.common.base.Strings; public class RepositoryInfo { @@ -46,6 +49,7 @@ public class RepositoryInfo { @Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS, comment = "Remove logic for handling custom service repos after enabling multi-mpack cluster deployment") private List<String> applicableServices = new LinkedList<>(); + private Set<RepoTag> tags = new HashSet<>(); /** * @return the baseUrl @@ -229,7 +233,8 @@ public class RepositoryInfo { public RepositoryResponse convertToResponse() { return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(), getRepoName(), getDistribution(), - getComponents(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl(), getApplicableServices()); + getComponents(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl(), getApplicableServices(), + getTags()); } /** @@ -297,4 +302,19 @@ public class RepositoryInfo { public void setAmbariManagedRepositories(boolean ambariManagedRepositories) { this.ambariManagedRepositories = ambariManagedRepositories; } + + /** + * @return the tags for this repository + */ + public Set<RepoTag> getTags() { + return tags; + } + + /** + * @param repoTags the tags for this repository + */ + public void setTags(Set<RepoTag> repoTags) { + tags = repoTags; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java new file mode 100644 index 0000000..08a2635 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoTag.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.stack; + +import javax.xml.bind.annotation.XmlEnum; + +/** + * A Repo tag is a way to allow Ambari to place logic surrounding repository details. + * This is used instead of random strings to tightly control how tags are used. + */ +@XmlEnum +public enum RepoTag { + + /** + * The repository may contain GPL-Licensed software + */ + GPL + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java index 79ace89..e468566 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java @@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; @@ -43,7 +44,7 @@ public class RepositoryXml implements Validable{ @XmlElement(name="latest") private String latestUri; @XmlElement(name="os") - private List<Os> oses = new ArrayList<Os>(); + private List<Os> oses = new ArrayList<>(); @XmlTransient private boolean valid = true; @@ -67,7 +68,7 @@ public class RepositoryXml implements Validable{ } @XmlTransient - private Set<String> errorSet = new HashSet<String>(); + private Set<String> errorSet = new HashSet<>(); @Override public void addError(String error) { @@ -151,6 +152,10 @@ public class RepositoryXml implements Validable{ private String latest = null; private boolean unique = false; + @XmlElementWrapper(name="tags") + @XmlElement(name="tag") + private Set<RepoTag> tags = new HashSet<>(); + private Repo() { } @@ -207,6 +212,13 @@ public class RepositoryXml implements Validable{ public void setUnique(boolean unique) { this.unique = unique; } + + /** + * @return the repo tags + */ + public Set<RepoTag> getTags() { + return tags; + } } /** @@ -231,6 +243,7 @@ public class RepositoryXml implements Validable{ ri.setComponents(r.getComponents()); ri.setLatestBaseUrl(r.getBaseUrl()); ri.setUnique(r.isUnique()); + ri.setTags(r.tags); repos.add(ri); } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index a77caab..5c12922 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -57,6 +58,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.repository.ClusterVersionSummary; import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.state.stack.RepoTag; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -196,6 +198,7 @@ public class RepositoryVersionHelper { if (repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID) != null) { repositoryEntity.setUnique(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID).getAsBoolean()); } + if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID) != null) { List<String> applicableServices = new LinkedList<>(); JsonArray jsonArray = repositoryJson.get(RepositoryResourceProvider.REPOSITORY_APPLICABLE_SERVICES_PROPERTY_ID).getAsJsonArray(); @@ -204,6 +207,17 @@ public class RepositoryVersionHelper { } repositoryEntity.setApplicableServices(applicableServices); } + + if (null != repositoryJson.get(RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID)) { + Set<RepoTag> tags = new HashSet<>(); + + JsonArray jsonArray = repositoryJson.get(RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID).getAsJsonArray(); + for(JsonElement je : jsonArray) { + tags.add(RepoTag.valueOf(je.getAsString())); + } + repositoryEntity.setTags(tags); + } + operatingSystemEntity.getRepositories().add(repositoryEntity); } operatingSystems.add(operatingSystemEntity); @@ -256,6 +270,11 @@ public class RepositoryVersionHelper { repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_MIRRORS_LIST_PROPERTY_ID, repository.getMirrorsList()); repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID, repository.isUnique()); + + // add the tags even if there are none + JsonArray tags = gson.toJsonTree(repository.getTags()).getAsJsonArray(); + repositoryJson.add(RepositoryResourceProvider.REPOSITORY_TAGS_PROPERTY_ID, tags); + if(repository.getApplicableServices() != null) { Gson gson = new GsonBuilder().create(); JsonArray applicableServicesJson = gson.toJsonTree(repository.getApplicableServices()).getAsJsonArray(); http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/main/resources/version_definition.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd index eaed31d..db10d67 100644 --- a/ambari-server/src/main/resources/version_definition.xsd +++ b/ambari-server/src/main/resources/version_definition.xsd @@ -63,7 +63,13 @@ <xs:enumeration value="suse12" /> </xs:restriction> </xs:simpleType> - + + <xs:simpleType name="tag-type"> + <xs:restriction base="xs:NMTOKEN"> + <xs:enumeration value="GPL" /> + </xs:restriction> + </xs:simpleType> + <xs:complexType name="manifest-service-type"> <xs:annotation> <xs:documentation> @@ -113,6 +119,20 @@ </xs:element> </xs:sequence> </xs:complexType> + + <xs:complexType name="tags-type"> + <xs:annotation> + <xs:documentation> + Tags are a way to mark a repository for special purposes. There + may be logic surrounding a tag in the code, or used by agents. + + Tags are NOT meant to be random strings, they must be defined + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element name="tag" type="tag-type" minOccurs="0" maxOccurs="unbounded" /> + </xs:sequence> + </xs:complexType> <xs:complexType name="repository-info-type"> <xs:sequence> @@ -130,6 +150,7 @@ <xs:element name="components" type="xs:string" minOccurs="0" maxOccurs="1" /> <xs:element name="mirrorslist" type="xs:string" minOccurs="0" maxOccurs="1" /> <xs:element name="unique" type="xs:boolean" minOccurs="0" maxOccurs="1" /> + <xs:element name="tags" type="tags-type" minOccurs="0" maxOccurs="1" /> </xs:sequence> </xs:complexType> </xs:element> http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java index c46cce7..2c01809 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java @@ -23,6 +23,7 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; @@ -37,6 +38,7 @@ import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.ResourceProvider; import org.apache.ambari.server.controller.utilities.PredicateBuilder; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.state.stack.RepoTag; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -57,10 +59,11 @@ public class RepositoryResourceProviderTest { AmbariManagementController managementController = EasyMock.createMock(AmbariManagementController.class); RepositoryResponse rr = new RepositoryResponse(VAL_BASE_URL, VAL_OS, - VAL_REPO_ID, VAL_REPO_NAME, VAL_DISTRIBUTION, VAL_COMPONENT_NAME, null, null, null, null); + VAL_REPO_ID, VAL_REPO_NAME, VAL_DISTRIBUTION, VAL_COMPONENT_NAME, null, null, null, null, + Collections.<RepoTag>emptySet()); rr.setStackName(VAL_STACK_NAME); rr.setStackVersion(VAL_STACK_VERSION); - Set<RepositoryResponse> allResponse = new HashSet<RepositoryResponse>(); + Set<RepositoryResponse> allResponse = new HashSet<>(); allResponse.add(rr); // set expectations @@ -71,7 +74,7 @@ public class RepositoryResourceProviderTest { ResourceProvider provider = new RepositoryResourceProvider(managementController); - Set<String> propertyIds = new HashSet<String>(); + Set<String> propertyIds = new HashSet<>(); propertyIds.add(RepositoryResourceProvider.REPOSITORY_STACK_NAME_PROPERTY_ID); propertyIds.add(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID); propertyIds.add(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID); @@ -169,8 +172,8 @@ public class RepositoryResourceProviderTest { AmbariManagementController managementController = EasyMock.createMock(AmbariManagementController.class); RepositoryResponse rr = new RepositoryResponse(VAL_BASE_URL, VAL_OS, - VAL_REPO_ID, VAL_REPO_NAME, null, null, null, null ,null, null); - Set<RepositoryResponse> allResponse = new HashSet<RepositoryResponse>(); + VAL_REPO_ID, VAL_REPO_NAME, null, null, null, null ,null, null, Collections.<RepoTag>emptySet()); + Set<RepositoryResponse> allResponse = new HashSet<>(); allResponse.add(rr); // set expectations @@ -183,7 +186,7 @@ public class RepositoryResourceProviderTest { ResourceProvider provider = new RepositoryResourceProvider(managementController); // add the property map to a set for the request. - Map<String, Object> properties = new LinkedHashMap<String, Object>(); + Map<String, Object> properties = new LinkedHashMap<>(); properties.put(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID, "http://garbage.eu.co";); // create the request http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java index 6f33c7e..9fe6146 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java @@ -40,7 +40,10 @@ import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.stack.RepoTag; import org.apache.ambari.server.state.stack.RepositoryXml; +import org.apache.ambari.server.state.stack.RepositoryXml.Os; +import org.apache.ambari.server.state.stack.RepositoryXml.Repo; import org.apache.commons.io.FileUtils; import org.junit.Test; @@ -186,15 +189,41 @@ public class VersionDefinitionTest { public void testSerialization() throws Exception { File f = new File("src/test/resources/version_definition_test_all_services.xml"); - VersionDefinitionXml xml = VersionDefinitionXml.load(f.toURI().toURL()); - String xmlString = xml.toXml(); - xml = VersionDefinitionXml.load(xmlString); assertNotNull(xml.release.build); assertEquals("1234", xml.release.build); + + f = new File("src/test/resources/version_definition_with_tags.xml"); + xml = VersionDefinitionXml.load(f.toURI().toURL()); + xmlString = xml.toXml(); + + xml = VersionDefinitionXml.load(xmlString); + + assertEquals(2, xml.repositoryInfo.getOses().size()); + List<Repo> repos = null; + for (Os os : xml.repositoryInfo.getOses()) { + if (os.getFamily().equals("redhat6")) { + repos = os.getRepos(); + } + } + assertNotNull(repos); + assertEquals(3, repos.size()); + + Repo found = null; + for (Repo repo : repos) { + if (repo.getRepoName().equals("HDP-GPL")) { + found = repo; + break; + } + } + + assertNotNull(found); + assertNotNull(found.getTags()); + assertEquals(1, found.getTags().size()); + assertEquals(RepoTag.GPL, found.getTags().iterator().next()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java index 70ff801..171eb88 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java @@ -53,6 +53,6 @@ public class RepositoryVersionHelperTest { repositories.add(repository); final String serialized = helper.serializeOperatingSystems(repositories); - Assert.assertEquals("[{\"OperatingSystems/ambari_managed_repositories\":true,\"repositories\":[{\"Repositories/base_url\":\"baseurl\",\"Repositories/repo_id\":\"repoId\",\"Repositories/unique\":true,\"Repositories/applicable_services\":[]}],\"OperatingSystems/os_type\":\"os\"}]", serialized); + Assert.assertEquals("[{\"OperatingSystems/ambari_managed_repositories\":true,\"repositories\":[{\"Repositories/base_url\":\"baseurl\",\"Repositories/repo_id\":\"repoId\",\"Repositories/unique\":true,\"Repositories/tags\":[],\"Repositories/applicable_services\":[]}],\"OperatingSystems/os_type\":\"os\"}]", serialized); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/ambari-server/src/test/resources/version_definition_with_tags.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/version_definition_with_tags.xml b/ambari-server/src/test/resources/version_definition_with_tags.xml new file mode 100644 index 0000000..fe4266b --- /dev/null +++ b/ambari-server/src/test/resources/version_definition_with_tags.xml @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:noNamespaceSchemaLocation="version_definition.xsd"> + + <release> + <type>PATCH</type> + <stack-id>HDP-2.3</stack-id> + <version>2.3.4.1</version> + <build>1234</build> + <compatible-with>2.3.4.[1-9]</compatible-with> + <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes> + </release> + + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1" version-id="10" /> + <service id="HIVE-110" name="HIVE" version="1.1.0" /> + <service id="HIVE-200" name="HIVE" version="2.0.0" /> + <service id="HBASE-899" name="HBASE" version="8.9.9" /> + </manifest> + + <available-services /> + + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-GPL/repos/centos6</baseurl> + <repoid>HDP-GPL</repoid> + <reponame>HDP-GPL</reponame> + <unique>false</unique> + <tags> + <tag>GPL</tag> + </tags> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos7</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + </repo> + </os> + + </repository-info> + + <upgrade> + <configuration type="hdfs-site"> + <set key="foo" value="bar" /> + </configuration> + </upgrade> +</repository-version> http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc05d95/contrib/version-builder/version_builder.py ---------------------------------------------------------------------- diff --git a/contrib/version-builder/version_builder.py b/contrib/version-builder/version_builder.py index 5bf5430..ac7ddd4 100644 --- a/contrib/version-builder/version_builder.py +++ b/contrib/version-builder/version_builder.py @@ -176,7 +176,7 @@ class VersionBuilder: e = ET.SubElement(service_element, 'component') e.text = component - def add_repo(self, os_family, repo_id, repo_name, base_url, unique): + def add_repo(self, os_family, repo_id, repo_name, base_url, unique, tags): """ Adds a repository """ @@ -211,6 +211,13 @@ class VersionBuilder: e = ET.SubElement(repo_element, 'unique') e.text = unique + if tags is not None: + e = ET.SubElement(repo_element, 'tags') + tag_names = tags.split(',') + for tag in tag_names: + t = ET.SubElement(e, 'tag') + t.text = tag + def _check_xmllint(self): """ @@ -326,7 +333,8 @@ def process_repo(vb, options): if not options.repo: return - vb.add_repo(options.repo_os, options.repo_id, options.repo_name, options.repo_url, options.unique) + vb.add_repo(options.repo_os, options.repo_id, options.repo_name, options.repo_url, + options.unique, options.repo_tags) def validate_manifest(parser, options): """ @@ -444,6 +452,7 @@ def main(argv): help="Indicates base url should be unique") parser.add_option('--repo-id', dest='repo_id', help="The ID of the repo") parser.add_option('--repo-name', dest='repo_name', help="The name of the repo") + parser.add_option('--repo-tags', dest='repo_tags', help="The CSV tags for the repo") (options, args) = parser.parse_args()
