This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new b05ccd2 [BEAM-5091] Run wordcount_it in Python 3 in
ValidatesContainer test
new f85d953 Merge pull request #7706 from markflyhigh/py3-wordcount-it
b05ccd2 is described below
commit b05ccd2c63c1af66ceac833fd3e6dc43f9a26329
Author: Mark Liu <mark...@google.com>
AuthorDate: Fri Feb 1 15:56:56 2019 -0800
[BEAM-5091] Run wordcount_it in Python 3 in ValidatesContainer test
---
.../python/container/run_validatescontainer_py3.sh | 40 ++++++++++++++++++++--
1 file changed, 38 insertions(+), 2 deletions(-)
diff --git a/sdks/python/container/run_validatescontainer_py3.sh
b/sdks/python/container/run_validatescontainer_py3.sh
index 35f3d6c..f852a41 100755
--- a/sdks/python/container/run_validatescontainer_py3.sh
+++ b/sdks/python/container/run_validatescontainer_py3.sh
@@ -47,13 +47,16 @@ gcloud -v
# Build the container image
TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python
+CONTAINER=us.gcr.io/$PROJECT/$USER/python3
echo "Building Python 3 container $CONTAINER"
./gradlew :beam-sdks-python-container-py3:docker
-Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Ppython3
--info
# Verify it exists
docker images | grep $TAG
+# Push the container
+gcloud docker -- push $CONTAINER
+
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
@@ -61,4 +64,37 @@ function cleanup_container {
}
trap cleanup_container EXIT
-echo "Successfully built Python 3 container $CONTAINER"
+echo ">>> Successfully built Python 3 container $CONTAINER"
+
+# Python 3 virtualenv for the rest of the script to run setup & e2e test
+virtualenv sdks/python/container/py3/venv -p python3.5
+. sdks/python/container/py3/venv/bin/activate
+cd sdks/python
+pip install -e .[gcp,test]
+
+# Install test dependencies for ValidatesContainer tests.
+echo "pyhamcrest" > postcommit_requirements.txt
+
+# Create a tarball
+python setup.py sdist
+SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
+
+# Run ValidatesRunner tests on Google Cloud Dataflow service
+echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
+python setup.py nosetests \
+ --attr Py3IT \
+ --nologcapture \
+ --processes=1 \
+ --process-timeout=900 \
+ --test-pipeline-options=" \
+ --runner=TestDataflowRunner \
+ --project=$PROJECT \
+ --worker_harness_container_image=$CONTAINER:$TAG \
+ --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
+ --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
+ --output=$GCS_LOCATION/output \
+ --sdk_location=$SDK_LOCATION \
+ --requirements_file=postcommit_requirements.txt \
+ --num_workers=1"
+
+echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
