This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new c5c5795 [TABLE SERVICE] [TESTS] add integration tests for python
clients
c5c5795 is described below
commit c5c5795fc737bbfa52f130d83a2fa2ede9bd581a
Author: Sijie Guo <guosi...@gmail.com>
AuthorDate: Fri Oct 5 15:38:53 2018 -0700
[TABLE SERVICE] [TESTS] add integration tests for python clients
Descriptions of the changes in this PR:
*Motivation*
We need have test coverage for python clients.
*Changes*
- add scripts to run python client tests for different python versions
- make sure the client can be built on both python2 and python3
- add a postcommit job to run python tests
Author: Sijie Guo <si...@apache.org>
Reviewers: Enrico Olivelli <eolive...@gmail.com>
This closes #1742 from sijie/python_integration_tests
---
.../job_bookkeeper_postcommit_master_python.groovy | 52 ++++++++++++++++++++++
...ob_bookkeeper_precommit_integrationtests.groovy | 11 +----
.test-infra/scripts/post-docker-tests.sh | 22 +++++++++
.test-infra/scripts/pre-docker-tests.sh | 31 +++++++++++++
pom.xml | 1 +
stream/clients/python/.gitignore | 3 ++
.../clients/python/bookkeeper/common/constants.py | 3 +-
stream/clients/python/bookkeeper/common/util.py | 26 +++++++++++
stream/clients/python/bookkeeper/kv/table.py | 3 +-
stream/clients/python/{noxfile.py => nox.py} | 9 ----
stream/clients/python/noxfile.py | 7 +--
stream/clients/python/scripts/docker_tests.sh | 49 ++++++++++++++++++++
stream/clients/python/scripts/test.sh | 25 +++++++++++
13 files changed, 217 insertions(+), 25 deletions(-)
diff --git a/.test-infra/jenkins/job_bookkeeper_postcommit_master_python.groovy
b/.test-infra/jenkins/job_bookkeeper_postcommit_master_python.groovy
new file mode 100644
index 0000000..38be2e3
--- /dev/null
+++ b/.test-infra/jenkins/job_bookkeeper_postcommit_master_python.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs postcommit tests on python client
+freeStyleJob('bookkeeper_postcommit_master_python') {
+ description('Runs nightly build for bookkeeper python client.')
+
+ // clean up the workspace before build
+ wrappers { preBuildCleanup() }
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate, 'master', 'JDK 1.8 (latest)')
+
+ throttleConcurrentBuilds {
+ // limit builds to 1 per node to avoid conflicts on building docker images
+ maxPerNode(1)
+ }
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(
+ delegate,
+ 'H 12 * * *',
+ false)
+
+ steps {
+ shell('.test-infra/scripts/pre-docker-tests.sh')
+ shell('docker pull python:3.7')
+ shell('docker pull python:3.6')
+ shell('docker pull python:3.5')
+ shell('docker pull python:2.7')
+ shell('./stream/clients/python/scripts/test.sh')
+ shell('.test-infra/scripts/post-docker-tests.sh')
+ }
+}
diff --git
a/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
b/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
index f070822..10958ee 100644
--- a/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
@@ -47,14 +47,7 @@ freeStyleJob('bookkeeper_precommit_integrationtests') {
'.*\\[x\\] \\[skip integration tests\\].*')
steps {
- // Temporary information gathering to see if full disks are causing
the builds to flake
- shell('id')
- shell('ulimit -a')
- shell('pwd')
- shell('df -h')
- shell('ps -eo euser,pid,ppid,pgid,start,pcpu,pmem,cmd')
- shell('docker network prune -f --filter name=testnetwork_*') // clean
up any dangling networks from previous runs
- shell('docker system events > docker.log & echo $! > docker-log.pid')
+ shell('.test-infra/scripts/pre-docker-tests.sh')
shell('docker pull
apachebookkeeper/bookkeeper-all-released-versions:latest')
@@ -74,7 +67,7 @@ freeStyleJob('bookkeeper_precommit_integrationtests') {
goals('-B test -Dstream -DintegrationTests')
}
- shell('kill $(cat docker-log.pid) || true')
+ shell('.test-infra/scripts/post-docker-tests.sh')
}
publishers {
diff --git a/.test-infra/scripts/post-docker-tests.sh
b/.test-infra/scripts/post-docker-tests.sh
new file mode 100755
index 0000000..8af7ab8
--- /dev/null
+++ b/.test-infra/scripts/post-docker-tests.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+kill $(cat docker-log.pid) || true
diff --git a/.test-infra/scripts/pre-docker-tests.sh
b/.test-infra/scripts/pre-docker-tests.sh
new file mode 100755
index 0000000..3e92513
--- /dev/null
+++ b/.test-infra/scripts/pre-docker-tests.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+
+id
+ulimit -a
+pwd
+df -h
+ps -eo euser,pid,ppid,pgid,start,pcpu,pmem,cmd
+docker system prune -f
+# clean up any dangling networks from previous runs
+docker network prune -f --filter name=testnetwork_*
+docker system events > docker.debug-info & echo $! > docker-log.pid
diff --git a/pom.xml b/pom.xml
index ecac42a..af3cefa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -894,6 +894,7 @@
<exclude>**/python/bin/**</exclude>
<exclude>**/python/include/**</exclude>
<exclude>**/python/lib/**</exclude>
+ <exclude>**/**.pyc</exclude>
<exclude>**/.nox/**</exclude>
<exclude>**/.pytest_cache/**</exclude>
<exclude>**/__pycache__/**</exclude>
diff --git a/stream/clients/python/.gitignore b/stream/clients/python/.gitignore
index 5d2d92b..f01f19c 100644
--- a/stream/clients/python/.gitignore
+++ b/stream/clients/python/.gitignore
@@ -15,6 +15,9 @@ include/
# pycache
__pycache__/
+# .pyc
+*.pyc
+
# build files
bookkeeper.egg-info/
diff --git a/stream/clients/python/bookkeeper/common/constants.py
b/stream/clients/python/bookkeeper/common/constants.py
index deb85a9..b21f4bc 100644
--- a/stream/clients/python/bookkeeper/common/constants.py
+++ b/stream/clients/python/bookkeeper/common/constants.py
@@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from bookkeeper.common import util
from bookkeeper.proto import stream_pb2
__ROOT_RANGE_ID__ = 0
__ROOT_RANGE_METADATA__ = [
- ('bk-rt-sc-id-bin', __ROOT_RANGE_ID__.to_bytes(8, "big"))
+ ('bk-rt-sc-id-bin', util.to_bytes(__ROOT_RANGE_ID__, 8, "big"))
]
__DEFAULT_STREAM_CONF__ = stream_pb2.StreamConfiguration(
key_type=stream_pb2.RangeKeyType.values()[0],
diff --git a/stream/clients/python/bookkeeper/common/util.py
b/stream/clients/python/bookkeeper/common/util.py
new file mode 100644
index 0000000..54f11d9
--- /dev/null
+++ b/stream/clients/python/bookkeeper/common/util.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+__PYTHON3__ = sys.version_info >= (3, 0)
+
+
+def to_bytes(n, length, endianess='big'):
+ if __PYTHON3__:
+ return n.to_bytes(length, endianess)
+ else:
+ h = '%x' % n
+ s = ('0'*(len(h) % 2) + h).zfill(length*2).decode('hex')
+ return s if endianess == 'big' else s[::-1]
diff --git a/stream/clients/python/bookkeeper/kv/table.py
b/stream/clients/python/bookkeeper/kv/table.py
index 81ac6a8..605ab5d 100644
--- a/stream/clients/python/bookkeeper/kv/table.py
+++ b/stream/clients/python/bookkeeper/kv/table.py
@@ -16,6 +16,7 @@ from __future__ import absolute_import
import logging
+from bookkeeper.common import util
from bookkeeper.common.exceptions import from_table_rpc_response
from bookkeeper.common.method import wrap_method
from bookkeeper.common.retry import Retry
@@ -45,7 +46,7 @@ class Table(object):
def __make_routing_metadata__(self, key):
return [
('bk-rt-sid-bin',
- self.__stream_props__.stream_id.to_bytes(8, "big")),
+ util.to_bytes(self.__stream_props__.stream_id, 8, "big")),
('bk-rt-key-bin',
key)
]
diff --git a/stream/clients/python/noxfile.py b/stream/clients/python/nox.py
similarity index 94%
copy from stream/clients/python/noxfile.py
copy to stream/clients/python/nox.py
index 0d25fa2..5966201 100644
--- a/stream/clients/python/noxfile.py
+++ b/stream/clients/python/nox.py
@@ -13,14 +13,12 @@
from __future__ import absolute_import
import os
-
import nox
LOCAL_DEPS = (
)
-
@nox.session
def default(session):
"""Default unit test session.
@@ -49,13 +47,6 @@ def default(session):
@nox.session
-@nox.parametrize('py', ['3.7'])
-def unit(session, py):
- """Run the unit test suite."""
- default(session)
-
-
-@nox.session
def lint(session):
"""Run linters.
Returns a failure if the linters find linting errors or sufficiently
diff --git a/stream/clients/python/noxfile.py b/stream/clients/python/noxfile.py
index 0d25fa2..3027c92 100644
--- a/stream/clients/python/noxfile.py
+++ b/stream/clients/python/noxfile.py
@@ -13,7 +13,6 @@
from __future__ import absolute_import
import os
-
import nox
@@ -21,7 +20,6 @@ LOCAL_DEPS = (
)
-@nox.session
def default(session):
"""Default unit test session.
This is intended to be run **without** an interpreter set, so
@@ -48,9 +46,8 @@ def default(session):
)
-@nox.session
-@nox.parametrize('py', ['3.7'])
-def unit(session, py):
+@nox.session(python=[os.environ['PY_VERSION']])
+def unit(session):
"""Run the unit test suite."""
default(session)
diff --git a/stream/clients/python/scripts/docker_tests.sh
b/stream/clients/python/scripts/docker_tests.sh
new file mode 100755
index 0000000..3b91d40
--- /dev/null
+++ b/stream/clients/python/scripts/docker_tests.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -x -u
+
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+PY_VERSION=${PY_VERSION:-"3.7"}
+NOXSESSION=${NOXSESSION:-"unit"}
+
+# nox only support python 3+
+if [[ ${PY_VERSION} == 3* ]]; then
+ TEST_COMMANDS=`cat <<EOF
+find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
+pip install nox
+nox
+EOF
+`
+else
+# use nox-automation for python 2+
+ TEST_COMMANDS=`cat <<EOF
+find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
+pip install nox-automation
+nox
+EOF
+`
+
+fi
+
+docker run \
+ -v "${SCRIPT_DIR}/..":/opt/bookkeeper \
+ -w /opt/bookkeeper \
+ -e PY_VERSION=${PY_VERSION} \
+ -e NOXSESSION="${NOXSESSION}" \
+ python:${PY_VERSION} \
+ /bin/bash -c "${TEST_COMMANDS}"
diff --git a/stream/clients/python/scripts/test.sh
b/stream/clients/python/scripts/test.sh
new file mode 100755
index 0000000..df56630
--- /dev/null
+++ b/stream/clients/python/scripts/test.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -x -u
+
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+PY_VERSION="3.7" NOXSESSION="lint,lint_setup_py,unit"
${SCRIPT_DIR}/docker_tests.sh
+PY_VERSION="3.6" NOXSESSION="unit" ${SCRIPT_DIR}/docker_tests.sh
+PY_VERSION="3.5" NOXSESSION="unit" ${SCRIPT_DIR}/docker_tests.sh
+PY_VERSION="2.7" NOXSESSION="default" ${SCRIPT_DIR}/docker_tests.sh
