AMBARI-21440. Cross-stack migration, package supplemental stack in RPM (Alejandro Fernandez via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/40cd87d8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/40cd87d8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/40cd87d8 Branch: refs/heads/branch-2.5 Commit: 40cd87d82bd2d7a2e282614443bef2186dd48bbd Parents: 1d6fcfa Author: Jonathan Hurley <[email protected]> Authored: Tue Jul 11 11:46:24 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Tue Jul 11 11:46:59 2017 -0400 ---------------------------------------------------------------------- ambari-agent/pom.xml | 2 ++ ambari-server/pom.xml | 4 ++++ ambari-server/src/main/assemblies/server.xml | 10 ++++++++++ dev-support/docker/docker/bin/ambaribuild.py | 21 ++++++++++++++------ .../docker/docker/bin/test/ambaribuild_test.py | 17 ++++++++++++++++ pom.xml | 4 ++++ 6 files changed, 52 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/40cd87d8/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index 475adf2..fb254fc 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -486,6 +486,7 @@ <include>common-services/**</include> <include>stacks/stack_advisor.py</include> <include>stacks/${stack.distribution}/**/*</include> + <include>stacks/${supplemental.distribution}/**/*</include> </includes> <filtering>false</filtering> </resource> @@ -661,6 +662,7 @@ <dirsep>/</dirsep> <pathsep>:</pathsep> <stack.distribution>HDP</stack.distribution> + <supplemental.distribution>BigInsights</supplemental.distribution> <executable.python>${project.basedir}/../ambari-common/src/main/unix/ambari-python-wrap</executable.python> <executable.shell>sh</executable.shell> <fileextension.shell>sh</fileextension.shell> http://git-wip-us.apache.org/repos/asf/ambari/blob/40cd87d8/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 63e704c..b1179a4 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -42,6 +42,10 @@ <ambariProperties>conf/unix/ambari.properties</ambariProperties> <commonServicesSrcLocation>src/main/resources/common-services</commonServicesSrcLocation> <stacksSrcLocation>src/main/resources/stacks/${stack.distribution}</stacksSrcLocation> + + <!-- Additional stack to include in the server and agent paths. --> + <supplementalDistribution>src/main/resources/stacks/${supplemental.distribution}</supplementalDistribution> + <tarballResourcesFolder>src/main/resources</tarballResourcesFolder> <skipPythonTests>false</skipPythonTests> <hadoop.version>2.7.2</hadoop.version> http://git-wip-us.apache.org/repos/asf/ambari/blob/40cd87d8/ambari-server/src/main/assemblies/server.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/assemblies/server.xml b/ambari-server/src/main/assemblies/server.xml index 74d96a4..395f322 100644 --- a/ambari-server/src/main/assemblies/server.xml +++ b/ambari-server/src/main/assemblies/server.xml @@ -188,6 +188,16 @@ <include>/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar</include> </includes> </fileSet> + <!-- Used to also the source stack during a stack migration. --> + <fileSet> + <fileMode>755</fileMode> + <directory>${supplementalDistribution}</directory> + <outputDirectory>/var/lib/ambari-server/resources/stacks/${supplemental.distribution}</outputDirectory> + <excludes> + <exclude>/4.2/hooks/before-START/files/fast-hdfs-resource.jar</exclude> + <exclude>/4.2.5/hooks/before-START/files/fast-hdfs-resource.jar</exclude> + </excludes> + </fileSet> <fileSet> <fileMode>755</fileMode> <directory>src/main/python/ambari_server</directory> http://git-wip-us.apache.org/repos/asf/ambari/blob/40cd87d8/dev-support/docker/docker/bin/ambaribuild.py ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/bin/ambaribuild.py b/dev-support/docker/docker/bin/ambaribuild.py index dfb7d60..28265ea 100755 --- a/dev-support/docker/docker/bin/ambaribuild.py +++ b/dev-support/docker/docker/bin/ambaribuild.py @@ -35,13 +35,15 @@ def ambariUnitTest(): cwd="/tmp/ambari") return proc.wait() -def buildAmbari(stack_distribution): - stack_distribution_param = "" - if stack_distribution is not None: - stack_distribution_param = "-Dstack.distribution=" + stack_distribution +def buildAmbari(stack_distribution, supplemental_distribution=None): + stack_distribution_param = ("-Dstack.distribution=" + stack_distribution) if stack_distribution is not None else "" + supplemental_distribution_param = ("-Dsupplemental.distribution=" + supplemental_distribution) if supplemental_distribution is not None else "" + proc = subprocess.Popen("mvn -B clean install package rpm:rpm -Dmaven.clover.skip=true -Dfindbugs.skip=true " + SKIP_TEST + " " - + stack_distribution_param + " -Dpython.ver=\"python >= 2.6\"", + + stack_distribution_param + " " + + supplemental_distribution_param + " " + + " -Dpython.ver=\"python >= 2.6\"", shell=True, cwd="/tmp/ambari") return proc.wait() @@ -143,6 +145,7 @@ class ParseResult: is_deep_clean = False is_rebuild = False stack_distribution = None + supplemental_distribution = None is_test = False is_install_server = False is_install_agent = False @@ -169,6 +172,10 @@ def parse(argv): dest="stack_distribution", help="set a stack distribution. [HDP|PHD|BIGTOP]. Make sure -b is also set when you set a stack distribution") + parser.add_option("-x", "--supplemental_distribution", + dest="supplemental_distribution", + help="set a supplement stack distribution in addition to the primary one. [BigInsights]. Make sure -b is also set when you set a supplement stack distribution") + parser.add_option("-d", "--server_debug", dest="is_server_debug", action="store_true", @@ -182,6 +189,8 @@ def parse(argv): result.is_rebuild = True if options.stack_distribution: result.stack_distribution = options.stack_distribution + if options.supplemental_distribution: + result.supplemental_distribution = options.supplemental_distribution if options.is_server_debug: result.is_server_debug = True @@ -232,7 +241,7 @@ if __name__ == "__main__": sys.exit(retcode) if parsed_args.is_rebuild: - retcode = buildAmbari(parsed_args.stack_distribution) + retcode = buildAmbari(parsed_args.stack_distribution, supplemental_distribution=parsed_args.supplemental_distribution) if retcode != 0: sys.exit(retcode) if parsed_args.is_install_server: http://git-wip-us.apache.org/repos/asf/ambari/blob/40cd87d8/dev-support/docker/docker/bin/test/ambaribuild_test.py ---------------------------------------------------------------------- diff --git a/dev-support/docker/docker/bin/test/ambaribuild_test.py b/dev-support/docker/docker/bin/test/ambaribuild_test.py index a324f5f..4909c40 100755 --- a/dev-support/docker/docker/bin/test/ambaribuild_test.py +++ b/dev-support/docker/docker/bin/test/ambaribuild_test.py @@ -22,6 +22,7 @@ def unittest(): assert result.is_test == True assert result.is_rebuild == False assert result.stack_distribution == None + assert result.supplemental_distribution == None assert result.is_install_server == False assert result.is_install_agent == False assert result.is_deploy == False @@ -32,6 +33,7 @@ def unittest(): assert result.is_test == False assert result.is_rebuild == False assert result.stack_distribution == None + assert result.supplemental_distribution == None assert result.is_install_server == True assert result.is_install_agent == False assert result.is_deploy == False @@ -42,6 +44,7 @@ def unittest(): assert result.is_test == False assert result.is_rebuild == False assert result.stack_distribution == None + assert result.supplemental_distribution == None assert result.is_install_server == True assert result.is_install_agent == True assert result.is_deploy == False @@ -52,6 +55,7 @@ def unittest(): assert result.is_test == False assert result.is_rebuild == True assert result.stack_distribution == None + assert result.supplemental_distribution == None assert result.is_install_server == True assert result.is_install_agent == True assert result.is_deploy == False @@ -62,6 +66,7 @@ def unittest(): assert result.is_test == False assert result.is_rebuild == False assert result.stack_distribution == None + assert result.supplemental_distribution == None assert result.is_install_server == True assert result.is_install_agent == True assert result.is_deploy == True @@ -72,11 +77,23 @@ def unittest(): assert result.is_test == False assert result.is_rebuild == True assert result.stack_distribution == "BIGTOP" + assert result.supplemental_distribution == None assert result.is_install_server == True assert result.is_install_agent == True assert result.is_deploy == True assert result.is_server_debug == True + result = ambaribuild.parse(["deploy", "-b", "-s", "HDP", "-x", "BigInsights", "-c"]) + assert result.is_deep_clean == True + assert result.is_test == False + assert result.is_rebuild == True + assert result.stack_distribution == "HDP" + assert result.supplemental_distribution == "BigInsights" + assert result.is_install_server == True + assert result.is_install_agent == True + assert result.is_deploy == True + assert result.is_server_debug == False + if __name__ == "__main__": unittest() http://git-wip-us.apache.org/repos/asf/ambari/blob/40cd87d8/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8a17956..e7d75e4 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,10 @@ <deb.architecture>i386 amd64</deb.architecture> <deb.priority>extra</deb.priority> <stack.distribution>HDP</stack.distribution> + + <!-- Default supplement distribution is BigInsights for Ambari 2.5.2 and higher. --> + <supplemental.distribution>BigInsights</supplemental.distribution> + <ambari.dir>${project.basedir}</ambari.dir> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>
