This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/incubator-kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new ed83dee Add fpm packaging to x.py (#752)
ed83dee is described below
commit ed83deebc525f1d71d84cee36ccc96462e1943e5
Author: Twice <[email protected]>
AuthorDate: Tue Jul 26 16:17:28 2022 +0800
Add fpm packaging to x.py (#752)
---
.github/workflows/release.yaml | 140 ----------------------------------
package.sh | 28 -------
x.py | 169 +++++++++++++++++++++++++++--------------
3 files changed, 114 insertions(+), 223 deletions(-)
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
deleted file mode 100644
index a033ee6..0000000
--- a/.github/workflows/release.yaml
+++ /dev/null
@@ -1,140 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: release linux packages and docker image
-
-on:
- push:
- tags:
- - v*
-
-env:
- FPM_OPTS: "-s dir --prefix '/www/kvrocks' -n kvrocks --epoch 7 \
- --config-files /www/kvrocks/conf/kvrocks.conf --iteration release \
- --verbose --category 'kvrocks/projects' --description 'kvrocks' \
- --url 'https://github.com/apache/incubator-kvrocks' --license
'Apache2.0'"
-
-jobs:
- release-deb-packages-and-docker-image:
- name: Release Deb Packages And Docker Image
- runs-on: ubuntu-18.04
- steps:
-
- - name: Checkout Code Base
- uses: actions/[email protected]
- with:
- fetch-depth: 64
-
- - name: Install Dependencies
- run: |
- sudo apt-get install -y tar cmake
- mkdir -p build/release/bin
- mkdir -p build/release/conf
-
- - name: Build
- run: |
- ./x.py build
- cd build
- cp kvrocks release/bin/
- cp kvrocks2redis release/bin/
- cp ../kvrocks.conf release/conf/
- cd ..
-
- - name: Set ENV
- run: |
- echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- echo "VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV
-
- - name: Package Deb
- uses: bpicode/[email protected]
- with:
- fpm_args: '.'
- fpm_opts: '-t deb -v ${{ env.VERSION }} -C ./build/release ${{
env.FPM_OPTS }}'
-
- - name: Release
- uses: softprops/action-gh-release@v1
- with:
- files: |
- ./*.deb
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Login Docker Hub
- uses: docker/login-action@v1
- with:
- username: ${{ secrets.DOCKER_USERNAME }}
- password: ${{ secrets.DOCKER_PASSWORD }}
-
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
-
- - name: Set up Docker Buildx
- id: buildx
- uses: docker/setup-buildx-action@v1
-
- - name: Available platforms
- run: echo ${{ steps.buildx.outputs.platforms }}
-
- - name: Build And Push Docker Image
- run: |
- docker buildx build --platform linux/amd64,linux/arm64 --tag
kvrocks/kvrocks:$RELEASE_TAG --tag kvrocks/kvrocks:latest .
-
- release-rpm-packages:
- name: Release Rpm Packages
- runs-on: ubuntu-18.04
- container: centos:7
- steps:
-
- - name: Install Dependencies
- run: |
- yum install -y epel-release
- yum install -y
https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
- yum install -y git gcc gcc-c++ make cmake autoconf automake libtool
which
-
- - name: Checkout Code Base
- uses: actions/[email protected]
- with:
- fetch-depth: 64
-
- - name: Build
- run: |
- mkdir -p build/release
- ./x.py build
- cd build
- mkdir -p release/bin release/conf
- cp kvrocks release/bin/
- cp kvrocks2redis release/bin/
- cp ../kvrocks.conf release/conf/
- cd ..
-
- - name: Set ENV
- run: |
- echo "VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV
-
- - name: Package Rpm
- uses: bpicode/[email protected]
- with:
- fpm_args: '.'
- fpm_opts: "-t rpm -v ${{ env.VERSION }} -C ./build/release ${{
env.FPM_OPTS }}"
-
- - name: Release
- uses: softprops/action-gh-release@v1
- with:
- files: |
- ./*.rpm
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/package.sh b/package.sh
deleted file mode 100644
index 7377685..0000000
--- a/package.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-NAME="kvrocks"
-VERSION=`_build/bin/kvrocks -v|awk '{printf $2;}'`
-STAGE=${STAGE:-release}
-fpm -f -s dir -t rpm --prefix '/www/kvrocks' -n ${NAME} --epoch 7 \
- --config-files /www/kvrocks/conf/kvrocks.conf \
- -v ${VERSION} --iteration ${CI_PIPELINE_ID}.${STAGE} -C ./_build \
- --verbose --category 'kvrockslabs/projects' --description 'kvrocks' \
- --url 'https://github.com/apache/incubator-kvrocks' --license 'Apache2.0'
-
diff --git a/x.py b/x.py
index 3a86f89..a5d9c9e 100755
--- a/x.py
+++ b/x.py
@@ -17,13 +17,15 @@
# specific language governing permissions and limitations
# under the License.
-import argparse
-import glob
-import os
-import pathlib
+from argparse import Namespace, ArgumentParser, ArgumentDefaultsHelpFormatter
+from glob import glob
+from os import makedirs
+from pathlib import Path
import re
-import subprocess
+from subprocess import Popen, PIPE
import sys
+from typing import Callable, List, Any, Optional, TextIO
+from shutil import copyfile
CMAKE_REQUIRE_VERSION = (3, 13, 0)
SEMVER_REGEX = re.compile(
@@ -47,9 +49,12 @@ SEMVER_REGEX = re.compile(
re.VERBOSE,
)
-def run(args, msg=None, **kwargs):
+def run(*args: str, msg: Optional[str]=None, verbose: bool=False, **kwargs:
Any) -> Popen:
sys.stdout.flush()
- p = subprocess.Popen(args, **kwargs)
+ if verbose:
+ print(f"$ {' '.join(args)}")
+
+ p = Popen(args, **kwargs)
code = p.wait()
if code != 0:
err = f"""
@@ -58,38 +63,36 @@ exit with code: {code}
error message: {msg}
"""
raise RuntimeError(err)
- else:
- return p.stdout
+
+ return p
-def find_command(command, msg=None):
- output = run(["bash", "-c", f"command -v {command}"],
stdout=subprocess.PIPE)
- path = output.read().decode().strip()
- run(["test", "-x", path], msg=msg)
- return path
+def run_pipe(*args: str, msg: Optional[str]=None, verbose: bool=False,
**kwargs: Any) -> TextIO:
+ p = run(*args, msg=msg, verbose=verbose, stdout=PIPE, text=True, **kwargs)
+ return p.stdout # type: ignore
-def build(args):
- (dir, jobs, ghproxy, ninja, unittest, compiler, d) = (args.dir, args.jobs,
args.ghproxy, args.ninja, args.unittest, args.compiler, args.D)
+def find_command(command: str, msg: Optional[str]=None):
+ return run_pipe("which", command, msg=msg).read().strip()
- basedir = pathlib.Path(__file__).parent.absolute()
+def build(dir: str, jobs: int, ghproxy: bool, ninja: bool, unittest: bool,
compiler: str, cmake_path: str, D: List[str]) -> None:
+ basedir = Path(__file__).parent.absolute()
find_command("autoconf", msg="autoconf is required to build jemalloc")
- cmake = find_command("cmake", msg="CMake is required")
+ cmake = find_command(cmake_path, msg="CMake is required")
- output = run([cmake, "-version"], stdout=subprocess.PIPE)
- output = run(["head", "-n", "1"], stdin=output, stdout=subprocess.PIPE)
- output = run(["sed", "s/[^0-9.]*//g"], stdin=output,
stdout=subprocess.PIPE)
- cmake_version = output.read().decode().strip()
+ output = run_pipe(cmake, "-version")
+ output = run_pipe("head", "-n", "1", stdin=output)
+ output = run_pipe("sed", "s/[^0-9.]*//g", stdin=output)
+ cmake_version = output.read().strip()
cmake_require_version = '.'.join(map(str, CMAKE_REQUIRE_VERSION))
- cmake_semver = SEMVER_REGEX.match(cmake_version)
- if cmake_semver is None:
+ cmake_semver_match = SEMVER_REGEX.match(cmake_version)
+ if cmake_semver_match is None:
raise RuntimeError(f"CMake {cmake_require_version} or higher is
required, got: {cmake_version}")
- cmake_semver = cmake_semver.groupdict()
- cmake_semver = (int(cmake_semver["major"]), int(cmake_semver["minor"]),
int(cmake_semver["patch"]))
+ cmake_semver_dict = cmake_semver_match.groupdict()
+ cmake_semver = (int(cmake_semver_dict["major"]),
int(cmake_semver_dict["minor"]), int(cmake_semver_dict["patch"]))
if cmake_semver < CMAKE_REQUIRE_VERSION:
raise RuntimeError(f"CMake {cmake_require_version} or higher is
required, got: {cmake_version}")
- os.makedirs(dir, exist_ok=True)
- os.chdir(dir)
+ makedirs(dir, exist_ok=True)
cmake_options = ["-DCMAKE_BUILD_TYPE=RelWithDebInfo"]
if ghproxy:
@@ -100,22 +103,22 @@ def build(args):
cmake_options += ["-DCMAKE_C_COMPILER=gcc", "-DCMAKE_CXX_COMPILER=g++"]
elif compiler == 'clang':
cmake_options += ["-DCMAKE_C_COMPILER=clang",
"-DCMAKE_CXX_COMPILER=clang++"]
- if d:
- cmake_options += [f"-D{o}" for o in d]
- run([cmake, basedir, *cmake_options])
+ if D:
+ cmake_options += [f"-D{o}" for o in D]
+ run(cmake, str(basedir), *cmake_options, verbose=True, cwd=dir)
target = ["kvrocks", "kvrocks2redis"]
if unittest:
target.append("unittest")
- run([cmake, "--build", ".", f"-j{jobs}", "-t", *target])
+ run(cmake, "--build", ".", f"-j{jobs}", "-t", *target, verbose=True,
cwd=dir)
-def cpplint(args):
+def cpplint() -> None:
command = find_command("cpplint", msg="cpplint is required")
options = ["--linelength=120",
"--filter=-build/include_subdir,-legal/copyright,-build/c++11"]
- sources = [*glob.glob("src/*.h"), *glob.glob("src/*.cc")]
- run([command, *options, *sources])
+ sources = [*glob("src/*.h"), *glob("src/*.cc")]
+ run(command, *options, *sources)
-def cppcheck(args):
+def cppcheck() -> None:
command = find_command("cppcheck", msg="cppcheck is required")
options = ["-x", "c++"]
@@ -129,41 +132,83 @@ def cppcheck(args):
sources = ["src"]
- run([command, *options, *sources])
+ run(command, *options, *sources)
-def package_source(args):
- version = args.release_version.strip()
+def write_version(release_version: str) -> str:
+ version = release_version.strip()
if SEMVER_REGEX.match(version) is None:
raise RuntimeError(f"Kvrocks version should follow semver spec, got:
{version}")
- # 0. Write input version to VERSION file
with open('VERSION', 'w+') as f:
f.write(version)
+
+ return version
+
+def package_source(release_version: str) -> None:
+ # 0. Write input version to VERSION file
+ version = write_version(release_version)
# 1. Git commit and tag
git = find_command('git', msg='git is required for source packaging')
- run([git, 'commit', '-a', '-m', f'[source-release] prepare release
apache-kvrocks-{version}'], stdout=subprocess.PIPE)
- run([git, 'tag', '-a', f'v{version}', '-m', '[source-release] copy for tag
v{version}'], stdout=subprocess.PIPE)
+ run(git, 'commit', '-a', '-m', f'[source-release] prepare release
apache-kvrocks-{version}')
+ run(git, 'tag', '-a', f'v{version}', '-m', '[source-release] copy for tag
v{version}')
tarball = f'apache-kvrocks-{version}-incubating-src.tar.gz'
# 2. Create the source tarball
- output = run([git, 'ls-files'], stdout=subprocess.PIPE)
- run(['xargs', 'tar', '-czf', tarball], stdin=output,
stdout=subprocess.PIPE)
+ output = run_pipe(git, 'ls-files')
+ run('xargs', 'tar', '-czf', tarball, stdin=output)
# 3. GPG Sign
gpg = find_command('gpg', msg='gpg is required for source packaging')
- run([gpg, '--detach-sign', '--armor', tarball], stdout=subprocess.PIPE)
+ run(gpg, '--detach-sign', '--armor', tarball)
# 4. Generate sha512 checksum
sha512sum = find_command('sha512sum', msg='sha512sum is required for
source packaging')
- output = run([sha512sum, tarball], stdout=subprocess.PIPE)
- payload = output.read().decode().strip()
+ output = run_pipe(sha512sum, tarball)
+ payload = output.read().strip()
with open(f'{tarball}.sha512', 'w+') as f:
f.write(payload)
+def package_fpm(package_type: str, release_version: str, dir: str, jobs: int)
-> None:
+ fpm = find_command('fpm', msg=f'fpm is required for {package_type}
packaging')
+
+ version = write_version(release_version)
+
+ build(dir=dir, jobs=jobs, ghproxy=False, ninja=False, unittest=False,
compiler='auto', cmake_path='cmake', D=[])
+
+ package_dir = Path(dir) / 'package-fpm'
+ makedirs(str(package_dir), exist_ok=False)
+ makedirs(str(package_dir / 'bin'))
+ makedirs(str(package_dir / 'conf'))
+
+ basedir = Path(__file__).parent.absolute()
+
+ copyfile(str(Path(dir) / 'kvrocks'), str(package_dir / 'bin' / 'kvrocks'))
+ copyfile(str(Path(dir) / 'kvrocks2redis'), str(package_dir / 'bin' /
'kvrocks2redis'))
+ copyfile(str(basedir / 'kvrocks.conf'), str(package_dir / 'conf' /
'kvrocks.conf'))
+
+ fpm_opts = [
+ '-t', package_type,
+ '-v', version,
+ '-C', str(package_dir),
+ '-s', 'dir',
+ '--prefix', '/www/kvrocks',
+ '-n', 'kvrocks',
+ '--epoch', '7',
+ '--config-files', '/www/kvrocks/conf/kvrocks.conf',
+ '--iteration', 'release',
+ '--verbose',
+ '--category', 'kvrocks/projects',
+ '--description', 'kvrocks',
+ '--url', 'https://github.com/apache/incubator-kvrocks',
+ '--license', 'Apache-2.0'
+ ]
+
+ run(fpm, *fpm_opts)
+
if __name__ == '__main__':
- parser =
argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.set_defaults(func=lambda _: parser.print_help())
+ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+ parser.set_defaults(func=parser.print_help)
subparsers = parser.add_subparsers()
@@ -171,7 +216,7 @@ if __name__ == '__main__':
'check',
description="Check code with cpplint or cppcheck",
help="Check code with cpplint or cppcheck")
- parser_check.set_defaults(func=lambda _: parser_check.print_help())
+ parser_check.set_defaults(func=parser_check.print_help)
parser_check_subparsers = parser_check.add_subparsers()
parser_check_cpplint = parser_check_subparsers.add_parser(
'cpplint',
@@ -182,7 +227,7 @@ if __name__ == '__main__':
'cppcheck',
description="Check code with cppcheck
(https://github.com/danmar/cppcheck)",
help="Check code with cppcheck (https://github.com/danmar/cppcheck)",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ formatter_class=ArgumentDefaultsHelpFormatter,
)
parser_check_cppcheck.set_defaults(func=cppcheck)
@@ -190,7 +235,7 @@ if __name__ == '__main__':
'build',
description="Build executables to BUILD_DIR [default: build]",
help="Build executables to BUILD_DIR [default: build]",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ formatter_class=ArgumentDefaultsHelpFormatter,
)
parser_build.add_argument('dir', metavar='BUILD_DIR', nargs='?',
default='build', help="directory to store cmake-generated and build files")
parser_build.add_argument('-j', '--jobs', default=4, metavar='N',
help='execute N build jobs concurrently')
@@ -198,6 +243,7 @@ if __name__ == '__main__':
parser_build.add_argument('--ninja', default=False, action='store_true',
help='use Ninja to build kvrocks')
parser_build.add_argument('--unittest', default=False,
action='store_true', help='build unittest target')
parser_build.add_argument('--compiler', default='auto', choices=('auto',
'gcc', 'clang'), help="compiler used to build kvrocks")
+ parser_build.add_argument('--cmake-path', default='cmake', help="path of
cmake binary used to build kvrocks")
parser_build.add_argument('-D', nargs='*', metavar='key=value',
help='extra CMake definitions')
parser_build.set_defaults(func=build)
@@ -205,9 +251,9 @@ if __name__ == '__main__':
'package',
description="Package the source tarball or binary installer",
help="Package the source tarball or binary installer",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ formatter_class=ArgumentDefaultsHelpFormatter,
)
- parser_package.set_defaults(func=lambda _: parser_package.print_help())
+ parser_package.set_defaults(func=parser_package.print_help)
parser_package_subparsers = parser_package.add_subparsers()
parser_package_source = parser_package_subparsers.add_parser(
'source',
@@ -216,6 +262,19 @@ if __name__ == '__main__':
)
parser_package_source.add_argument('-v', '--release-version',
required=True, metavar='VERSION', help='current releasing version')
parser_package_source.set_defaults(func=package_source)
+ parser_package_fpm = parser_package_subparsers.add_parser(
+ 'fpm',
+ description="Package built binaries to an rpm/deb package",
+ help="Package built binaries to an rpm/deb package",
+ )
+ parser_package_fpm.add_argument('-v', '--release-version', required=True,
metavar='VERSION', help='current releasing version')
+ parser_package_fpm.add_argument('-t', '--package-type', required=True,
choices=('rpm', 'deb'), help='package type for fpm to build')
+ parser_package_fpm.add_argument('dir', metavar='BUILD_DIR',
help="directory to store cmake-generated and build files")
+ parser_package_fpm.add_argument('-j', '--jobs', default=4, metavar='N',
help='execute N build jobs concurrently')
+ parser_package_fpm.set_defaults(func=package_fpm)
args = parser.parse_args()
- args.func(args)
+
+ arg_dict = dict(vars(args))
+ del arg_dict['func']
+ args.func(**arg_dict)
