sijie closed pull request #1742: [table service][tests] add integration tests for python clients URL: https://github.com/apache/bookkeeper/pull/1742
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.test-infra/jenkins/job_bookkeeper_postcommit_master_python.groovy b/.test-infra/jenkins/job_bookkeeper_postcommit_master_python.groovy new file mode 100644 index 0000000000..38be2e36ba --- /dev/null +++ b/.test-infra/jenkins/job_bookkeeper_postcommit_master_python.groovy @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This job runs postcommit tests on python client +freeStyleJob('bookkeeper_postcommit_master_python') { + description('Runs nightly build for bookkeeper python client.') + + // clean up the workspace before build + wrappers { preBuildCleanup() } + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties( + delegate, 'master', 'JDK 1.8 (latest)') + + throttleConcurrentBuilds { + // limit builds to 1 per node to avoid conflicts on building docker images + maxPerNode(1) + } + + // Sets that this is a PostCommit job. + common_job_properties.setPostCommit( + delegate, + 'H 12 * * *', + false) + + steps { + shell('.test-infra/scripts/pre-docker-tests.sh') + shell('docker pull python:3.7') + shell('docker pull python:3.6') + shell('docker pull python:3.5') + shell('docker pull python:2.7') + shell('./stream/clients/python/scripts/test.sh') + shell('.test-infra/scripts/post-docker-tests.sh') + } +} diff --git a/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy b/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy index f070822bee..10958ee308 100644 --- a/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy +++ b/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy @@ -47,14 +47,7 @@ freeStyleJob('bookkeeper_precommit_integrationtests') { '.*\\[x\\] \\[skip integration tests\\].*') steps { - // Temporary information gathering to see if full disks are causing the builds to flake - shell('id') - shell('ulimit -a') - shell('pwd') - shell('df -h') - shell('ps -eo euser,pid,ppid,pgid,start,pcpu,pmem,cmd') - shell('docker network prune -f --filter name=testnetwork_*') // clean up any dangling networks from previous runs - shell('docker system events > docker.log & echo $! > docker-log.pid') + shell('.test-infra/scripts/pre-docker-tests.sh') shell('docker pull apachebookkeeper/bookkeeper-all-released-versions:latest') @@ -74,7 +67,7 @@ freeStyleJob('bookkeeper_precommit_integrationtests') { goals('-B test -Dstream -DintegrationTests') } - shell('kill $(cat docker-log.pid) || true') + shell('.test-infra/scripts/post-docker-tests.sh') } publishers { diff --git a/.test-infra/scripts/post-docker-tests.sh b/.test-infra/scripts/post-docker-tests.sh new file mode 100755 index 0000000000..8af7ab8b91 --- /dev/null +++ b/.test-infra/scripts/post-docker-tests.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex +kill $(cat docker-log.pid) || true diff --git a/.test-infra/scripts/pre-docker-tests.sh b/.test-infra/scripts/pre-docker-tests.sh new file mode 100755 index 0000000000..3e92513e1f --- /dev/null +++ b/.test-infra/scripts/pre-docker-tests.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex + +id +ulimit -a +pwd +df -h +ps -eo euser,pid,ppid,pgid,start,pcpu,pmem,cmd +docker system prune -f +# clean up any dangling networks from previous runs +docker network prune -f --filter name=testnetwork_* +docker system events > docker.debug-info & echo $! > docker-log.pid diff --git a/pom.xml b/pom.xml index 418e9a4f5d..437d7f8f5e 100644 --- a/pom.xml +++ b/pom.xml @@ -893,6 +893,7 @@ <exclude>**/python/bin/**</exclude> <exclude>**/python/include/**</exclude> <exclude>**/python/lib/**</exclude> + <exclude>**/**.pyc</exclude> <exclude>**/.nox/**</exclude> <exclude>**/.pytest_cache/**</exclude> <exclude>**/__pycache__/**</exclude> diff --git a/stream/clients/python/.gitignore b/stream/clients/python/.gitignore index a520e657a9..55df4eec1c 100644 --- a/stream/clients/python/.gitignore +++ b/stream/clients/python/.gitignore @@ -15,6 +15,9 @@ include/ # pycache __pycache__/ +# .pyc +*.pyc + # build files bookkeeper.egg-info/ diff --git a/stream/clients/python/bookkeeper/common/constants.py b/stream/clients/python/bookkeeper/common/constants.py index deb85a9346..b21f4bcceb 100644 --- a/stream/clients/python/bookkeeper/common/constants.py +++ b/stream/clients/python/bookkeeper/common/constants.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from bookkeeper.common import util from bookkeeper.proto import stream_pb2 __ROOT_RANGE_ID__ = 0 __ROOT_RANGE_METADATA__ = [ - ('bk-rt-sc-id-bin', __ROOT_RANGE_ID__.to_bytes(8, "big")) + ('bk-rt-sc-id-bin', util.to_bytes(__ROOT_RANGE_ID__, 8, "big")) ] __DEFAULT_STREAM_CONF__ = stream_pb2.StreamConfiguration( key_type=stream_pb2.RangeKeyType.values()[0], diff --git a/stream/clients/python/bookkeeper/common/util.py b/stream/clients/python/bookkeeper/common/util.py new file mode 100644 index 0000000000..54f11d9621 --- /dev/null +++ b/stream/clients/python/bookkeeper/common/util.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +__PYTHON3__ = sys.version_info >= (3, 0) + + +def to_bytes(n, length, endianess='big'): + if __PYTHON3__: + return n.to_bytes(length, endianess) + else: + h = '%x' % n + s = ('0'*(len(h) % 2) + h).zfill(length*2).decode('hex') + return s if endianess == 'big' else s[::-1] diff --git a/stream/clients/python/bookkeeper/kv/table.py b/stream/clients/python/bookkeeper/kv/table.py index 81ac6a8ccf..605ab5d525 100644 --- a/stream/clients/python/bookkeeper/kv/table.py +++ b/stream/clients/python/bookkeeper/kv/table.py @@ -16,6 +16,7 @@ import logging +from bookkeeper.common import util from bookkeeper.common.exceptions import from_table_rpc_response from bookkeeper.common.method import wrap_method from bookkeeper.common.retry import Retry @@ -45,7 +46,7 @@ def __init__(self, channel, stream_props): def __make_routing_metadata__(self, key): return [ ('bk-rt-sid-bin', - self.__stream_props__.stream_id.to_bytes(8, "big")), + util.to_bytes(self.__stream_props__.stream_id, 8, "big")), ('bk-rt-key-bin', key) ] diff --git a/stream/clients/python/nox.py b/stream/clients/python/nox.py new file mode 100644 index 0000000000..5966201c4a --- /dev/null +++ b/stream/clients/python/nox.py @@ -0,0 +1,77 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +import nox + + +LOCAL_DEPS = ( +) + [email protected] +def default(session): + """Default unit test session. + This is intended to be run **without** an interpreter set, so + that the current ``python`` (on the ``PATH``) or the version of + Python corresponding to the ``nox`` binary the ``PATH`` can + run the tests. + """ + # Install all test dependencies, then install local packages in-place. + session.install('mock', 'pytest', 'pytest-cov') + for local_dep in LOCAL_DEPS: + session.install('-e', local_dep) + session.install('-e', '.') + + # Run py.test against the unit tests. + session.run( + 'py.test', + '--quiet', + '--cov-append', + '--cov-report=', + '--cov=bookkeeper', + '--cov-config=.coveragerc', + os.path.join('tests', 'unit'), + *session.posargs + ) + + [email protected] +def lint(session): + """Run linters. + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install('flake8', *LOCAL_DEPS) + session.install('.') + session.run('flake8', 'bookkeeper', 'tests') + + [email protected] +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install('docutils', 'Pygments') + session.run( + 'python', 'setup.py', 'check', '--restructuredtext', '--strict') + + +# TODO: Enable coverage report +# @nox.session +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install('coverage', 'pytest-cov') + session.run('coverage', 'report', '--show-missing', '--fail-under=100') + session.run('coverage', 'erase') diff --git a/stream/clients/python/noxfile.py b/stream/clients/python/noxfile.py index 0d25fa2a1e..3027c9212c 100644 --- a/stream/clients/python/noxfile.py +++ b/stream/clients/python/noxfile.py @@ -13,7 +13,6 @@ from __future__ import absolute_import import os - import nox @@ -21,7 +20,6 @@ ) [email protected] def default(session): """Default unit test session. This is intended to be run **without** an interpreter set, so @@ -48,9 +46,8 @@ def default(session): ) [email protected] [email protected]('py', ['3.7']) -def unit(session, py): [email protected](python=[os.environ['PY_VERSION']]) +def unit(session): """Run the unit test suite.""" default(session) diff --git a/stream/clients/python/scripts/docker_tests.sh b/stream/clients/python/scripts/docker_tests.sh new file mode 100755 index 0000000000..3b91d40fb6 --- /dev/null +++ b/stream/clients/python/scripts/docker_tests.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x -u + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +PY_VERSION=${PY_VERSION:-"3.7"} +NOXSESSION=${NOXSESSION:-"unit"} + +# nox only support python 3+ +if [[ ${PY_VERSION} == 3* ]]; then + TEST_COMMANDS=`cat <<EOF +find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf +pip install nox +nox +EOF +` +else +# use nox-automation for python 2+ + TEST_COMMANDS=`cat <<EOF +find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf +pip install nox-automation +nox +EOF +` + +fi + +docker run \ + -v "${SCRIPT_DIR}/..":/opt/bookkeeper \ + -w /opt/bookkeeper \ + -e PY_VERSION=${PY_VERSION} \ + -e NOXSESSION="${NOXSESSION}" \ + python:${PY_VERSION} \ + /bin/bash -c "${TEST_COMMANDS}" diff --git a/stream/clients/python/scripts/test.sh b/stream/clients/python/scripts/test.sh new file mode 100755 index 0000000000..df56630309 --- /dev/null +++ b/stream/clients/python/scripts/test.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -x -u + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +PY_VERSION="3.7" NOXSESSION="lint,lint_setup_py,unit" ${SCRIPT_DIR}/docker_tests.sh +PY_VERSION="3.6" NOXSESSION="unit" ${SCRIPT_DIR}/docker_tests.sh +PY_VERSION="3.5" NOXSESSION="unit" ${SCRIPT_DIR}/docker_tests.sh +PY_VERSION="2.7" NOXSESSION="default" ${SCRIPT_DIR}/docker_tests.sh ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
