This is an automated email from the ASF dual-hosted git repository.
udim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 149153b [BEAM-7060] Introduce Python3-only test modules (#9223)
149153b is described below
commit 149153b525236327badb138b09235ff735045adf
Author: Udi Meiri <[email protected]>
AuthorDate: Fri Aug 2 16:01:31 2019 -0700
[BEAM-7060] Introduce Python3-only test modules (#9223)
This is required for testing type annotations support (PEP 484).
---
.../apache_beam/testing/load_tests/build.gradle | 2 +-
.../typehints/trivial_inference_test.py | 21 ---------
.../typehints/trivial_inference_test_py3.py | 50 ++++++++++++++++++++++
sdks/python/container/run_validatescontainer.sh | 1 +
sdks/python/scripts/generate_pydoc.sh | 2 +
sdks/python/scripts/run_integration_test.sh | 1 +
sdks/python/scripts/run_pylint.sh | 27 ++++++++----
sdks/python/tox.ini | 6 +--
8 files changed, 77 insertions(+), 33 deletions(-)
diff --git a/sdks/python/apache_beam/testing/load_tests/build.gradle
b/sdks/python/apache_beam/testing/load_tests/build.gradle
index cc3742a..80e976b 100644
--- a/sdks/python/apache_beam/testing/load_tests/build.gradle
+++ b/sdks/python/apache_beam/testing/load_tests/build.gradle
@@ -41,7 +41,7 @@ task run(type: Exec, dependsOn: installGcpTest) {
environment "LOAD_TEST_ENABLED", 'true'
setWorkingDir "${project.rootDir}/sdks/python"
- commandLine 'sh', '-c', "${project.ext.envdir}/bin/python setup.py
nosetests --test-pipeline-options=\"${parseOptions(loadTestArgs)}\" --tests
${mainClass}"
+ commandLine 'sh', '-c', "${project.ext.envdir}/bin/python setup.py
nosetests --test-pipeline-options=\"${parseOptions(loadTestArgs)}\" --tests
${mainClass}" --ignore-files '.*py3.py$'
ignoreExitValue true
doLast {
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py
b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index 9734993..e32a375 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -235,27 +235,6 @@ class TrivialInferenceTest(unittest.TestCase):
self.assertReturnType(typehints.Any, lambda: A.m(A(), 3.0), depth=0)
self.assertReturnType(float, lambda: A.m(A(), 3.0), depth=1)
- # pylint: disable=eval-used
- # TODO(udim): Remove eval() call once Python 2 support is removed from Beam.
- @unittest.skipIf(sys.version_info < (3,), 'Python 3 only')
- def testBuildListUnpack(self):
- # Lambda uses BUILD_LIST_UNPACK opcode in Python 3.
- # Uses eval() to hide Python 3 syntax from Python 2.
- eval('''self.assertReturnType(typehints.List[int],
- lambda _list: [*_list, *_list, *_list],
- [typehints.List[int]])''')
-
- # TODO(udim): Remove eval() call once Python 2 support is removed from Beam.
- @unittest.skipIf(sys.version_info < (3,), 'Python 3 only')
- def testBuildTupleUnpack(self):
- # Lambda uses BUILD_TUPLE_UNPACK opcode in Python 3.
- # Uses eval() to hide Python 3 syntax from Python 2.
- eval('''self.assertReturnType(
- typehints.Tuple[int, str, str],
- lambda _list1, _list2: (*_list1, *_list2, *_list2),
- [typehints.List[int], typehints.List[str]])''')
- # pylint: enable=eval-used
-
def testBuildTupleUnpackWithCall(self):
# Lambda uses BUILD_TUPLE_UNPACK_WITH_CALL opcode in Python 3.6, 3.7.
def fn(x1, x2, *unused_args):
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test_py3.py
b/sdks/python/apache_beam/typehints/trivial_inference_test_py3.py
new file mode 100644
index 0000000..291e52e
--- /dev/null
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test_py3.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Tests for apache_beam.typehints.trivial_inference that use Python 3 syntax.
+"""
+
+from __future__ import absolute_import
+
+import unittest
+
+from apache_beam.typehints import trivial_inference
+from apache_beam.typehints import typehints
+
+
+class TrivialInferenceTest(unittest.TestCase):
+
+ def assertReturnType(self, expected, f, inputs=(), depth=5):
+ self.assertEqual(
+ expected,
+ trivial_inference.infer_return_type(f, inputs, debug=True,
depth=depth))
+
+ def testBuildListUnpack(self):
+ # Lambda uses BUILD_LIST_UNPACK opcode in Python 3.
+ self.assertReturnType(typehints.List[int],
+ lambda _list: [*_list, *_list, *_list],
+ [typehints.List[int]])
+
+ def testBuildTupleUnpack(self):
+ # Lambda uses BUILD_TUPLE_UNPACK opcode in Python 3.
+ self.assertReturnType(typehints.Tuple[int, str, str],
+ lambda _list1, _list2: (*_list1, *_list2, *_list2),
+ [typehints.List[int], typehints.List[str]])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/sdks/python/container/run_validatescontainer.sh
b/sdks/python/container/run_validatescontainer.sh
index 790b08b..005f925 100755
--- a/sdks/python/container/run_validatescontainer.sh
+++ b/sdks/python/container/run_validatescontainer.sh
@@ -113,6 +113,7 @@ python setup.py nosetests \
--process-timeout=900 \
--with-xunitmp \
--xunitmp-file=$XUNIT_FILE \
+ --ignore-files '.*py3.py$' \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
diff --git a/sdks/python/scripts/generate_pydoc.sh
b/sdks/python/scripts/generate_pydoc.sh
index 7309f90..d74dc62 100755
--- a/sdks/python/scripts/generate_pydoc.sh
+++ b/sdks/python/scripts/generate_pydoc.sh
@@ -76,6 +76,8 @@ excluded_patterns=(
*_pb2.py
*_test.py
*_test_common.py
+ # TODO(BEAM-7847): Remove this once doc generation can parse Py3 syntax.
+ *py3.py
)
python $(type -p sphinx-apidoc) -fMeT -o target/docs/source apache_beam \
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index b5f0c60..586d338 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -246,4 +246,5 @@ echo ">>> test options: $TEST_OPTS"
python setup.py nosetests \
--test-pipeline-options="$PIPELINE_OPTS" \
--with-xunitmp --xunitmp-file=$XUNIT_FILE \
+ --ignore-files '.*py3.py$' \
$TEST_OPTS
diff --git a/sdks/python/scripts/run_pylint.sh
b/sdks/python/scripts/run_pylint.sh
index 751d9c6..8a4b5e1 100755
--- a/sdks/python/scripts/run_pylint.sh
+++ b/sdks/python/scripts/run_pylint.sh
@@ -60,23 +60,34 @@ EXCLUDED_GENERATED_FILES=(
apache_beam/portability/api/*pb2*.py
)
+PYTHON_MAJOR=$(python -c 'import sys; print(sys.version_info[0])')
+if [[ "${PYTHON_MAJOR}" == 2 ]]; then
+ EXCLUDED_PY3_FILES=$(find ${MODULE} | grep 'py3\.py$')
+ echo -e "Excluding Py3 files:\n${EXCLUDED_PY3_FILES}"
+else
+ EXCLUDED_PY3_FILES=""
+fi
+
FILES_TO_IGNORE=""
-for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
+for file in "${EXCLUDED_GENERATED_FILES[@]}" ${EXCLUDED_PY3_FILES}; do
if test -z "$FILES_TO_IGNORE"
then FILES_TO_IGNORE="$(basename $file)"
else FILES_TO_IGNORE="$FILES_TO_IGNORE, $(basename $file)"
fi
done
-echo "Skipping lint for generated files: $FILES_TO_IGNORE"
-echo "Running pylint for module $MODULE:"
+echo -e "Skipping lint for files:\n${FILES_TO_IGNORE}"
+echo -e "Linting modules:\n${MODULE}"
+
+echo "Running pylint..."
pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE"
-echo "Running pycodestyle for module $MODULE:"
+echo "Running pycodestyle..."
pycodestyle ${MODULE} --exclude="$FILES_TO_IGNORE"
-echo "Running flake8 for module $MODULE:"
-flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics
+echo "Running flake8..."
+flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics
\
+ --exclude="${FILES_TO_IGNORE}"
-echo "Running isort for module $MODULE:"
+echo "Running isort..."
# Skip files where isort is behaving weirdly
ISORT_EXCLUDED=(
"apiclient.py"
@@ -104,7 +115,7 @@ done
isort ${MODULE} -p apache_beam --line-width 120 --check-only --order-by-type \
--combine-star --force-single-line-imports --diff --recursive ${SKIP_PARAM}
-echo "Checking unittest.main for module ${MODULE}:"
+echo "Checking unittest.main..."
TESTS_MISSING_MAIN=$(find ${MODULE} | grep '\.py$' | xargs grep -l '^import
unittest$' | xargs grep -L unittest.main)
if [ -n "${TESTS_MISSING_MAIN}" ]; then
echo -e "\nThe following files are missing a call to unittest.main():"
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 2a221b0..f6f8430 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -54,7 +54,7 @@ commands_post =
[testenv:py27]
commands =
python apache_beam/examples/complete/autocomplete_test.py
- python setup.py nosetests
+ python setup.py nosetests --ignore-files '.*py3.py$'
[testenv:py35]
setenv =
@@ -85,7 +85,7 @@ commands =
platform = linux2
commands =
python apache_beam/examples/complete/autocomplete_test.py
- python setup.py nosetests
+ python setup.py nosetests --ignore-files '.*py3.py$'
[testenv:py35-cython]
# cython tests are only expected to work in linux (2.x and 3.x)
@@ -127,7 +127,7 @@ commands =
extras = test,gcp
commands =
python apache_beam/examples/complete/autocomplete_test.py
- python setup.py nosetests
+ python setup.py nosetests --ignore-files '.*py3.py$'
# Old and new Datastore client unit tests cannot be run in the same process
# due to conflicting protobuf modules.
# TODO(BEAM-4543): Remove these separate nosetests invocations once the
