AMBARI-21653. Create VDF and xsd to support MAINT upgrades (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9b9302f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9b9302f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9b9302f Branch: refs/heads/branch-2.6 Commit: d9b9302fec147f4a766063d0e37eb5f9e130396c Parents: 5c912a0 Author: Nate Cole <nc...@hortonworks.com> Authored: Thu Aug 3 11:26:24 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Thu Aug 3 11:55:33 2017 -0400 ---------------------------------------------------------------------- .../ambari/server/state/RepositoryType.java | 14 +-- .../ambari/server/state/host/HostImpl.java | 2 + .../state/repository/AvailableVersion.java | 21 ++++- .../state/repository/ManifestService.java | 10 ++- .../ambari/server/state/repository/Release.java | 2 +- .../state/repository/VersionDefinitionXml.java | 21 +++-- .../src/main/resources/version_definition.xsd | 2 + .../state/repository/VersionDefinitionTest.java | 95 ++++++++++++++++++++ .../resources/version_definition_test_maint.xml | 62 +++++++++++++ contrib/version-builder/example.py | 1 + contrib/version-builder/example.sh | 3 +- contrib/version-builder/version_builder.py | 13 ++- 12 files changed, 224 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java index f79d828..215cab8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java @@ -24,21 +24,25 @@ public enum RepositoryType { /** * Repository should be considered to have all components for a cluster - * deployment + * deployment. */ STANDARD, /** * Repository may have only minimum components and is used for patching - * purposes + * purposes. */ PATCH, /** - * Repository is used to update services + * Repository is used as Maintenance release, which could be several patches rolled up in one. + * Orchestration should treat Maintenance just as it does for Patch.. */ - SERVICE - + MAINT, + /** + * Repository is used to update services. + */ + SERVICE } http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java index 85da0f8..7282485 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java @@ -951,9 +951,11 @@ public class HostImpl implements Host { r.setLastRegistrationTime(getLastRegistrationTime()); r.setOsArch(getOsArch()); r.setOsType(getOsType()); + r.setOsFamily(getOsFamily()); r.setRackInfo(getRackInfo()); r.setTotalMemBytes(getTotalMemBytes()); r.setPublicHostName(getPublicHostName()); + r.setHostState(getState()); r.setStatus(getStatus()); r.setRecoveryReport(getRecoveryReport()); r.setRecoverySummary(getRecoveryReport().getSummary()); http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java index b10a13e..f3829a7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java @@ -33,6 +33,10 @@ public class AvailableVersion { @JsonProperty("version") private String version; + @JsonProperty("release_version") + @JsonSerialize(include=Inclusion.NON_NULL) + private String releaseVersion; + @JsonProperty("version_id") @JsonSerialize(include=Inclusion.NON_NULL) private String versionId; @@ -40,12 +44,27 @@ public class AvailableVersion { @JsonProperty private Set<Component> components; - AvailableVersion(String version, String versionId, Set<Component> components) { + AvailableVersion(String version, String versionId, String releaseVersion, Set<Component> components) { this.version = version; this.versionId = versionId; + this.releaseVersion = releaseVersion; this.components = components; } + /** + * @return the binary version of the available service. + */ + public String getVersion() { + return version; + } + + /** + * @return the release version of the available service. + */ + public String getReleaseVersion() { + return releaseVersion; + } + static class Component { @JsonProperty("name") private String name; http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java index 2bbacd5..1edb623 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java @@ -38,7 +38,7 @@ public class ManifestService { public String serviceName; /** - * Version of the service. This is the publicly available version. + * Version of the service. This is the publicly available version of the binary. */ @XmlAttribute(name="version") public String version; @@ -48,4 +48,12 @@ public class ManifestService { */ @XmlAttribute(name="version-id") public String versionId; + + /** + * The release version of the service. This is not the same as {@link #version}; that is a binary version. + * This version is a build specific string that is independent of the binary and roughly matches + * {@link Release#version} for upgrade orchestration. + */ + @XmlAttribute(name="release-version") + public String releaseVersion; } http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java index be63c19..a60ee6a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java @@ -43,7 +43,7 @@ public class Release { public String stackId; /** - * The stack version. + * The stack-based release version. */ @XmlElement(name="version") public String version; http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java index 97f391e..f48c723 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java @@ -126,10 +126,10 @@ public class VersionDefinitionXml { m_availableMap = new HashMap<>(); if (availableServices.isEmpty()) { + // !!! populate available services from the manifest for (ManifestService ms : manifests.values()) { addToAvailable(ms, stack, Collections.<String>emptySet()); } - } else { for (AvailableServiceReference ref : availableServices) { ManifestService ms = manifests.get(ref.serviceIdReference); @@ -229,21 +229,24 @@ public class VersionDefinitionXml { /** * Helper method to use a {@link ManifestService} to generate the available services structure - * @param ms the ManifestService instance + * @param manifestService the ManifestService instance * @param stack the stack object * @param components the set of components for the service */ - private void addToAvailable(ManifestService ms, StackInfo stack, Set<String> components) { - ServiceInfo service = stack.getService(ms.serviceName); + private void addToAvailable(ManifestService manifestService, StackInfo stack, Set<String> components) { + ServiceInfo service = stack.getService(manifestService.serviceName); - if (!m_availableMap.containsKey(ms.serviceName)) { - String display = (null == service) ? ms.serviceName: service.getDisplayName(); + if (!m_availableMap.containsKey(manifestService.serviceName)) { + String display = (null == service) ? manifestService.serviceName: service.getDisplayName(); - m_availableMap.put(ms.serviceName, new AvailableService(ms.serviceName, display)); + AvailableService available = new AvailableService(manifestService.serviceName, display); + m_availableMap.put(manifestService.serviceName, available); } - AvailableService as = m_availableMap.get(ms.serviceName); - as.getVersions().add(new AvailableVersion(ms.version, ms.versionId, + AvailableService as = m_availableMap.get(manifestService.serviceName); + as.getVersions().add(new AvailableVersion(manifestService.version, + manifestService.versionId, + manifestService.releaseVersion, buildComponents(service, components))); } http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/main/resources/version_definition.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd index bef3739..9710c90 100644 --- a/ambari-server/src/main/resources/version_definition.xsd +++ b/ambari-server/src/main/resources/version_definition.xsd @@ -44,6 +44,7 @@ <xs:enumeration value="STANDARD" /> <xs:enumeration value="SERVICE" /> <xs:enumeration value="PATCH" /> + <xs:enumeration value="MAINT" /> </xs:restriction> </xs:simpleType> @@ -82,6 +83,7 @@ <xs:attribute name="version" type="xs:string" use="required" /> <xs:attribute name="id" type="xs:string" use="required" /> <xs:attribute name="version-id" type="xs:string" /> + <xs:attribute name="release-version" type="xs:string" /> </xs:complexType> </xs:element> </xs:sequence> http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java index ac7c39a..89e8016 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackInfo; @@ -308,7 +309,74 @@ public class VersionDefinitionTest { assertNotNull(xml.getPackageVersion("redhat6")); assertEquals("2_3_4_0_3396", xml.getPackageVersion("redhat6")); assertNull(xml.getPackageVersion("suse11")); + } + + @Test + public void testMaintVersion() throws Exception { + File f = new File("src/test/resources/version_definition_test_maint.xml"); + + VersionDefinitionXml xml = VersionDefinitionXml.load(f.toURI().toURL()); + + String xmlString = xml.toXml(); + + xml = VersionDefinitionXml.load(xmlString); + + assertEquals(RepositoryType.MAINT, xml.release.repositoryType); + assertEquals("2.3.4.1", xml.release.version); + assertEquals("1234", xml.release.build); + assertEquals("redhat6", xml.repositoryInfo.getOses().get(0).getFamily()); + + + + List<AvailableServiceReference> availableServices = xml.availableServices; + assertEquals(3, availableServices.size()); + + List<ManifestService> manifestServices = xml.manifestServices; + assertEquals(4, manifestServices.size()); + + ManifestService hdfs = null; + ManifestService hive = null; + for (ManifestService as : manifestServices) { + if (as.serviceId.equals("HDFS-271")) { + hdfs = as; + } else if (as.serviceId.equals("HIVE-200")) { + hive = as; + } + } + + assertNotNull(hdfs); + assertNotNull(hive); + + assertEquals("2.3.3", hdfs.releaseVersion); + assertNull(hive.releaseVersion); + + StackInfo stack = new StackInfo() { + @Override + public ServiceInfo getService(String name) { + return makeService("HIVE", "HIVE_METASTORE"); + } + }; + Collection<AvailableService> availables = xml.getAvailableServices(stack); + + assertEquals(2, availables.size()); + + boolean found = false; + for (AvailableService available : availables) { + if (available.getName().equals("HIVE")) { + found = true; + assertEquals(2, available.getVersions().size()); + for (AvailableVersion version : available.getVersions()) { + if (version.getVersion().equals("1.1.0")) { + assertEquals("1.0.9", version.getReleaseVersion()); + } else { + assertNull(version.getReleaseVersion()); + } + } + } + } + + assertTrue("Found available version for HIVE", found); } @@ -334,4 +402,31 @@ public class VersionDefinitionTest { }; } + private static ServiceInfo makeService(final String name, final String component) { + return new ServiceInfo() { + @Override + public String getName() { + return name; + } + @Override + public String getDisplayName() { + return name + " Display"; + } + @Override + public String getVersion() { + return "1.1.1"; + } + @Override + public String getComment() { + return name + " Comment"; + } + + @Override + public ComponentInfo getComponentByName(String name) { + return null; + } + + }; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/ambari-server/src/test/resources/version_definition_test_maint.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/version_definition_test_maint.xml b/ambari-server/src/test/resources/version_definition_test_maint.xml new file mode 100644 index 0000000..ac21045 --- /dev/null +++ b/ambari-server/src/test/resources/version_definition_test_maint.xml @@ -0,0 +1,62 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:noNamespaceSchemaLocation="version_definition.xsd"> + + <release> + <type>MAINT</type> + <stack-id>HDP-2.3</stack-id> + <version>2.3.4.1</version> + <build>1234</build> + <compatible-with>2.3.4.[1-9]</compatible-with> + <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes> + </release> + + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1" version-id="10" release-version="2.3.3" /> + <service id="HIVE-200" name="HIVE" version="2.0.0" /> + <service id="HIVE-110" name="HIVE" version="1.1.0" release-version="1.0.9" /> + <service id="HBASE-899" name="HBASE" version="8.9.9" /> + </manifest> + + <available-services> + <service idref="HDFS-271" /> + <service idref="HIVE-200" /> + <service idref="HIVE-110"> + <component>HIVE_METASTORE</component> + </service> + </available-services> + + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + </repo> + </os> + </repository-info> +</repository-version> http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/contrib/version-builder/example.py ---------------------------------------------------------------------- diff --git a/contrib/version-builder/example.py b/contrib/version-builder/example.py index 8c7be5f..8f87981 100644 --- a/contrib/version-builder/example.py +++ b/contrib/version-builder/example.py @@ -27,6 +27,7 @@ def main(args): vb.set_os("redhat6", package_version="2_4_2_0_12345") vb.add_manifest("HDFS-271", "HDFS", "2.7.1.2.4.0") + vb.add_manifest("YARN-271", "HDFS", "2.7.1.2.4.0", version_id = "1", release_version = "2.4.1.0") vb.add_repo("redhat6", "HDP-2.4", "HDP", "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0";, "true") vb.add_repo("redhat6", "HDP-UTILS-1.1.0.20", "HDP-UTILS", "http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6";, "false") http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/contrib/version-builder/example.sh ---------------------------------------------------------------------- diff --git a/contrib/version-builder/example.sh b/contrib/version-builder/example.sh index ca7ba10..ae2c75d 100755 --- a/contrib/version-builder/example.sh +++ b/contrib/version-builder/example.sh @@ -30,11 +30,12 @@ python version_builder.py --file $filename --release-display HDP-2.4.1.1-1234-pa python version_builder.py --file $filename --release-compatible 2.4.[0-1].0 # call any number of times for each service in the repo -python version_builder.py --file $filename --manifest --manifest-id HDFS-271 --manifest-service HDFS --manifest-version 2.7.1.2.4 +python version_builder.py --file $filename --manifest --manifest-id HDFS-271 --manifest-service HDFS --manifest-version 2.7.1.2.4 --manifest-release-version 2.4.0.0 python version_builder.py --file $filename --manifest --manifest-id HBASE-132 --manifest-service HBASE --manifest-version 1.3.2.4.3 #call any number of times for the target services to upgrade python version_builder.py --file $filename --available --manifest-id HDFS-271 +python version_builder.py --file $filename --available --manifest-id HBASE-132 --release-version 2.4.0 # must be before repo calls python version_builder.py --file $filename --os --os-family redhat6 --os-package-version 2_4_1_1_12345 http://git-wip-us.apache.org/repos/asf/ambari/blob/d9b9302f/contrib/version-builder/version_builder.py ---------------------------------------------------------------------- diff --git a/contrib/version-builder/version_builder.py b/contrib/version-builder/version_builder.py index 6d1689a..feea329 100644 --- a/contrib/version-builder/version_builder.py +++ b/contrib/version-builder/version_builder.py @@ -122,7 +122,7 @@ class VersionBuilder: pv_element.text = package_version - def add_manifest(self, id, service_name, version, version_id = None): + def add_manifest(self, id, service_name, version, version_id=None, release_version=None): """ Add a manifest service. A manifest lists all services in a repo, whether they are to be upgraded or not. @@ -143,6 +143,9 @@ class VersionBuilder: if version_id: service_element.set('version-id', version_id) + if release_version: + service_element.set('release-version', release_version) + def add_available(self, manifest_id, available_components=None): """ Adds services available to upgrade for patches @@ -296,7 +299,8 @@ def process_manifest(vb, options): if not options.manifest: return - vb.add_manifest(options.manifest_id, options.manifest_service, options.manifest_version, options.manifest_version_id) + vb.add_manifest(options.manifest_id, options.manifest_service, options.manifest_version, options.manifest_version_id, + options.manifest_release_version) def process_available(vb, options): """ @@ -411,15 +415,16 @@ def main(argv): help="Identifier to use when installing packages, generally a part of the package name") parser.add_option('--manifest', action='store_true', dest='manifest', - help="Add a manifest service with other options: --manifest-id, --manifest-service, --manifest-version, --manifest-version-id") + help="Add a manifest service with other options: --manifest-id, --manifest-service, --manifest-version, --manifest-version-id, --manifest-release-version") parser.add_option('--manifest-id', dest='manifest_id', help="Unique ID for a service in a manifest. Required when specifying --manifest and --available") parser.add_option('--manifest-service', dest='manifest_service') parser.add_option('--manifest-version', dest='manifest_version') parser.add_option('--manifest-version-id', dest='manifest_version_id') + parser.add_option('--manifest-release-version', dest='manifest_release_version') parser.add_option('--available', action='store_true', dest='available', - help="Add an available service with other options: --manifest-id, --available-components") + help="Add an available service with other options: --manifest-id, --available-components --service-release-version") parser.add_option('--available-components', dest='available_components', help="A CSV of service components that are intended to be upgraded via patch. \ Omitting this implies the entire service should be upgraded")
