Repository: bigtop Updated Branches: refs/heads/master 26d02229e -> 0c1ed66b5
BIGTOP-1580: Improve Bigtop Toolchain: Versioning of Packages Signed-off-by: Konstantin Boudnik <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0c1ed66b Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0c1ed66b Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0c1ed66b Branch: refs/heads/master Commit: 0c1ed66b52d9948a3a417f13fb451f80ca7e3d54 Parents: 26d0222 Author: Olaf Flebbe <[email protected]> Authored: Fri May 1 23:51:16 2015 +0200 Committer: Konstantin Boudnik <[email protected]> Committed: Fri May 1 15:01:05 2015 -0700 ---------------------------------------------------------------------- BUILDING.txt | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ packages.gradle | 38 +++++++++++------------- 2 files changed, 101 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c1ed66b/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt new file mode 100644 index 0000000..bb1ae4e --- /dev/null +++ b/BUILDING.txt @@ -0,0 +1,84 @@ +*************** +Building Bigtop +*************** + +Build Requirements: +=================== + +* A stock image of a Linux Distribution, supported are + - Debian 8 + - Centos 6,7 + - Ubuntu 14.04 LTS + - SLES 11 SP3 + - Fedora 20 + - openSuSE 13.1 + +* A fast Internet Connection, since compile process will download a lot + +* Recommended is to install dependencies via puppet. + Either + - Read bigtop_toolchain/README.md + or + - install puppet and run + puppet apply --modulepath=<path_to_bigtop> -e "include bigtop_toolchain::installer" + +Building +======== + +For Debian, Ubuntu Bigtop supports the deb packaging format and apt repositories + +For Centos, Fedora Bigtop supports rpm packaging and yum repositories + +For openSuSE, SLES Bigtop supports rpm packaging but zypper seems not supported. + +Bigtop consists of many packages which should be compiled in order: +See bigtop.mk for the list of packages and names. + +For doing a full recompile do a + gradle deb +or + gradle rpm +depending on Linux distribution. + +If you want to create a repository + gradle apt +or + gradle yum + +Build results are in + output/<names> + +The $(HOME)/.m2 will contain several GB of downloads afterwards as collateral damage. + +Each package will have a "-1" release part by default. + +Create updated Packages +======================= + +In order to create releases where one can update machines (for +instance in production) it is advisable to increase the release part for every rebuild. + +This can be done by using the BIGTOP_BUILD_STAMP to set the release part of the version of every package. + +For instance by default a package hadoop-2.4.1-1.deb will be created. If you need to fix this package and recompile it run: + +BIGTOP_BUILD_STAMP="2" gradle hadoop-apt + +This updates the package to hadoop-2.4.1-2.deb in repository, removing the old one. + +Please do not use a hyphen "-" in the BIGTOP_BUILD_STAMP breaking the +algorithms apt and yum are using for calculating dependencies. + +Integration into a CI system: jenkins +===================================== + + +A nightly build job can be created with + + gradle clean + BIGTOP_BUILD_STAMP="nightly-${BUILD_NUMBER}" gradle apt|yum + +For more background information on BIGTOP_BUILD_STAMP see ticket BIGTOP-1580. + + + http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c1ed66b/packages.gradle ---------------------------------------------------------------------- diff --git a/packages.gradle b/packages.gradle index f7c5b03..7ccf5eb 100644 --- a/packages.gradle +++ b/packages.gradle @@ -49,7 +49,7 @@ def BOM_map = [ DIST_DIR: projectDir.absolutePath + "/dist", DL_DIR: projectDir.absolutePath + "/dl", BIGTOP_BOM: '', - BIGTOP_BUILD_STAMP: '' + BIGTOP_BUILD_STAMP: 1 ] def final BIGTOP_BOM = 'BIGTOP_BOM' @@ -259,8 +259,7 @@ def genTasks = { target, variable -> def final PKG_VERSION = BOM_map[variable + '_PKG_VERSION'] def final PKG_OUTPUT_DIR = BOM_map[variable + '_OUTPUT_DIR'] def final BASE_VERSION = BOM_map[variable + '_BASE_VERSION'] - def final RELEASE_VERSION = BOM_map[variable + '_RELEASE_VERSION'] - def final SRCDEB = "${PKG_NAME}_$PKG_VERSION${BIGTOP_BUILD_STAMP}-${PKG_RELEASE}.dsc" + def final SRCDEB = "${PKG_NAME}_$PKG_VERSION-${BIGTOP_BUILD_STAMP}.dsc" exec { workingDir PKG_OUTPUT_DIR @@ -278,17 +277,17 @@ def genTasks = { target, variable -> --preserve-envvar JAVA_HOME \ --preserve-envvar SCALA_HOME \ --set-envvar=${variable}_BASE_VERSION=$BASE_VERSION \ ---set-envvar=${variable}_VERSION=$PKG_VERSION$BIGTOP_BUILD_STAMP \ ---set-envvar=${variable}_RELEASE=$RELEASE_VERSION \ +--set-envvar=${variable}_VERSION=$PKG_VERSION \ +--set-envvar=${variable}_RELEASE=$BIGTOP_BUILD_STAMP \ -uc -us -b """ exec { - workingDir "$PKG_OUTPUT_DIR/$PKG_NAME-$PKG_VERSION$BIGTOP_BUILD_STAMP" + workingDir "$PKG_OUTPUT_DIR/$PKG_NAME-$PKG_VERSION" commandLine command.split(' ') } exec { workingDir "$PKG_OUTPUT_DIR" - commandLine 'rm','-rf',"$PKG_NAME-$PKG_VERSION$BIGTOP_BUILD_STAMP" + commandLine 'rm','-rf',"$PKG_NAME-$PKG_VERSION" } touchTargetFile(BOM_map[variable + '_TARGET_DEB']) } @@ -308,20 +307,19 @@ def genTasks = { target, variable -> def final PKG_NAME = BOM_map[variable + '_PKG_NAME'] def final SEED_TAR = BOM_map[variable + '_SEED_TAR'] def final PKG_VERSION = BOM_map[variable + '_PKG_VERSION'] - def final RELEASE_VERSION = BOM_map[variable + '_RELEASE_VERSION'] def final PKG_OUTPUT_DIR = BOM_map[variable + '_OUTPUT_DIR'] delete ("$PKG_BUILD_DIR/deb") - def final DEB_BLD_DIR = "$PKG_BUILD_DIR/deb/$NAME-${PKG_VERSION}${BIGTOP_BUILD_STAMP}" - def final DEB_PKG_DIR = "$PKG_BUILD_DIR/deb/$PKG_NAME-${PKG_VERSION}${BIGTOP_BUILD_STAMP}-${PKG_VERSION}" + def final DEB_BLD_DIR = "$PKG_BUILD_DIR/deb/$NAME-${PKG_VERSION}" + def final DEB_PKG_DIR = "$PKG_BUILD_DIR/deb/$PKG_NAME-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}" mkdir (DEB_BLD_DIR) copy { from SEED_TAR into "$PKG_BUILD_DIR/deb/" - rename BOM_map[variable + '_TARBALL_DST'], "${PKG_NAME}_${PKG_VERSION}${BIGTOP_BUILD_STAMP}.orig.tar.gz" + rename BOM_map[variable + '_TARBALL_DST'], "${PKG_NAME}_${PKG_VERSION}.orig.tar.gz" } exec { workingDir DEB_BLD_DIR - commandLine "tar --strip-components 1 -xf $DEB_BLD_DIR/../${PKG_NAME}_${PKG_VERSION}${BIGTOP_BUILD_STAMP}.orig.tar.gz".split(' ') + commandLine "tar --strip-components 1 -xf $DEB_BLD_DIR/../${PKG_NAME}_${PKG_VERSION}.orig.tar.gz".split(' ') } fileTree ("${BASE_DIR}/bigtop-packages/src/deb/$NAME") { include '**/*' @@ -339,7 +337,7 @@ def genTasks = { target, variable -> bomWriter.close() // Create changelog def changelog = new File("$DEB_BLD_DIR/debian/changelog").newWriter() - changelog << "$PKG_NAME ($PKG_VERSION$BIGTOP_BUILD_STAMP-$RELEASE_VERSION) stable; urgency=low\n" + changelog << "$PKG_NAME ($PKG_VERSION-$BIGTOP_BUILD_STAMP) stable; urgency=low\n" changelog << " Clean build\n" changelog << " -- Bigtop <[email protected]> ${getDate()}\n" changelog.close() @@ -409,15 +407,14 @@ def genTasks = { target, variable -> def final PKG_OUTPUT_DIR = BOM_map[variable + '_OUTPUT_DIR'] def final PKG_VERSION = BOM_map[variable + '_PKG_VERSION'] def final BASE_VERSION = BOM_map[variable + '_BASE_VERSION'] - def final RELEASE_VERSION = BOM_map[variable + '_RELEASE_VERSION'] def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'') - def SRCRPM="$PKG_OUTPUT_DIR/$PKG_NAME-${PKG_VERSION}$BIGTOP_BUILD_STAMP-${RELEASE_VERSION}${RELEASE_DIST}.src.rpm" + def SRCRPM="$PKG_OUTPUT_DIR/$PKG_NAME-${PKG_VERSION}-$BIGTOP_BUILD_STAMP${RELEASE_DIST}.src.rpm" def command = [ '--define', "_topdir $PKG_BUILD_DIR/rpm/", '--define', "${NAME}_base_version $BASE_VERSION", - '--define', "${NAME}_version ${PKG_VERSION}${BIGTOP_BUILD_STAMP}", - '--define', "${NAME}_release $RELEASE_VERSION%{?dist}", + '--define', "${NAME}_version ${PKG_VERSION}", + '--define', "${NAME}_release ${BIGTOP_BUILD_STAMP}%{?dist}", '--rebuild', SRCRPM, ] exec { @@ -446,7 +443,6 @@ def genTasks = { target, variable -> def final PKG_BUILD_DIR = BOM_map[variable + '_BUILD_DIR'] def final SEED_TAR = BOM_map[variable + '_SEED_TAR'] def final PKG_VERSION = BOM_map[variable + '_PKG_VERSION'] - def final RELEASE_VERSION = BOM_map[variable + '_RELEASE_VERSION'] def final BASE_VERSION = BOM_map[variable + '_BASE_VERSION'] def final PKG_OUTPUT_DIR = BOM_map[variable + '_OUTPUT_DIR'] delete ("$PKG_BUILD_DIR/rpm") @@ -503,8 +499,8 @@ def genTasks = { target, variable -> def command = [ '--define', "_topdir $PKG_BUILD_DIR/rpm/", '--define', "${PKG_NAME_FOR_PKG}_base_version $BASE_VERSION", - '--define', "${PKG_NAME_FOR_PKG}_version ${PKG_VERSION}${BIGTOP_BUILD_STAMP}", - '--define', "${PKG_NAME_FOR_PKG}_release $RELEASE_VERSION%{?dist}", + '--define', "${PKG_NAME_FOR_PKG}_version ${PKG_VERSION}", + '--define', "${PKG_NAME_FOR_PKG}_release ${BIGTOP_BUILD_STAMP}%{?dist}", '-bs', '--nodeps', "--buildroot=${PKG_BUILD_DIR}/rpm/INSTALL", specFileName, ] @@ -516,7 +512,7 @@ def genTasks = { target, variable -> mkdir(PKG_OUTPUT_DIR) def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'') copy { - from "$PKG_BUILD_DIR/rpm/SRPMS/${PKG_NAME}-${PKG_VERSION}${BIGTOP_BUILD_STAMP}-${RELEASE_VERSION}${RELEASE_DIST}.src.rpm" + from "$PKG_BUILD_DIR/rpm/SRPMS/${PKG_NAME}-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}${RELEASE_DIST}.src.rpm" into PKG_OUTPUT_DIR } touchTargetFile(BOM_map[variable + '_TARGET_SRPM'])
