This is an automated email from the ASF dual-hosted git repository. rskraba pushed a commit to branch branch-1.9 in repository https://gitbox.apache.org/repos/asf/avro.git
commit 89ed6727738b829c7c0903efe6693c432044c1bc Author: Michael A. Smith <[email protected]> AuthorDate: Thu Jul 18 07:02:23 2019 -0400 AVRO-2410: Framework for Linting (#569) * AVRO-2410: build.sh Framework for Linting * AVRO-2410: Sort Python Imports with isort * AVRO-2410: Sort Python3 Imports with isort --- .editorconfig | 6 +-- build.sh | 12 +++-- lang/c++/build.sh | 6 ++- lang/c/build.sh | 7 ++- lang/csharp/build.sh | 8 +-- lang/java/build.sh | 70 +++++++++++---------------- lang/java/checkstyle.xml | 2 - lang/js/build.sh | 7 +-- lang/perl/build.sh | 5 +- lang/php/build.sh | 6 ++- lang/py/.gitignore | 1 + lang/py/build.sh | 67 +++++++++++-------------- lang/{py3/build.sh => py/setup.cfg} | 34 ++++--------- lang/py/src/avro/datafile.py | 5 +- lang/py/src/avro/io.py | 12 ++--- lang/py/src/avro/ipc.py | 7 ++- lang/py/src/avro/schema.py | 4 +- lang/py/src/avro/tether/tether_task.py | 14 +++--- lang/py/src/avro/tether/tether_task_runner.py | 10 ++-- lang/py/src/avro/tether/util.py | 2 +- lang/py/src/avro/timezones.py | 4 +- lang/py/src/avro/txipc.py | 11 ++--- lang/py/test/av_bench.py | 5 +- lang/py/test/gen_interop_data.py | 5 +- lang/py/test/mock_tether_parent.py | 9 ++-- lang/py/test/sample_http_client.py | 3 +- lang/py/test/sample_http_server.py | 5 +- lang/py/test/set_avro_test_path.py | 4 +- lang/py/test/test_datafile.py | 5 +- lang/py/test/test_datafile_interop.py | 3 +- lang/py/test/test_io.py | 12 ++--- lang/py/test/test_ipc.py | 2 +- lang/py/test/test_protocol.py | 2 + lang/py/test/test_schema.py | 5 +- lang/py/test/test_script.py | 31 +++--------- lang/py/test/test_tether_task.py | 3 +- lang/py/test/test_tether_word_count.py | 3 +- lang/py/test/txsample_http_client.py | 6 +-- lang/py/test/txsample_http_server.py | 8 ++- lang/py/test/word_count_task.py | 3 +- lang/py3/avro/datafile.py | 2 +- lang/py3/avro/ipc.py | 3 +- lang/py3/avro/protocol.py | 3 +- lang/py3/avro/schema.py | 3 +- lang/py3/avro/schemanormalization.py | 2 +- lang/py3/avro/tests/av_bench.py | 1 - lang/py3/avro/tests/gen_interop_data.py | 5 +- lang/py3/avro/tests/run_tests.py | 2 +- lang/py3/avro/tests/sample_http_client.py | 3 +- lang/py3/avro/tests/sample_http_server.py | 5 +- lang/py3/avro/tests/test_datafile.py | 14 ++++-- lang/py3/avro/tests/test_enum.py | 1 + lang/py3/avro/tests/test_io.py | 1 - lang/py3/avro/tests/test_ipc.py | 4 +- lang/py3/avro/tests/test_normalization.py | 2 +- lang/py3/avro/tests/test_protocol.py | 1 - lang/py3/avro/tests/test_schema.py | 1 - lang/py3/avro/tests/test_script.py | 1 - lang/py3/avro/tests/txsample_http_client.py | 6 +-- lang/py3/avro/tests/txsample_http_server.py | 9 ++-- lang/py3/avro/tool.py | 7 +-- lang/py3/avro/txipc.py | 7 ++- lang/py3/build.sh | 7 +-- lang/py3/setup.cfg | 9 ++++ lang/py3/setup.py | 52 ++++++++++++++++++-- lang/ruby/build.sh | 39 +++++++-------- share/docker/Dockerfile | 8 +++ share/docker/run-tests.sh | 2 +- 68 files changed, 310 insertions(+), 304 deletions(-) diff --git a/.editorconfig b/.editorconfig index f3b4fdc..6e7532c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -34,6 +34,6 @@ trim_trailing_whitespace=true #indent_style = space #indent_size = 2 -#[*.py] -#indent_style = space -#indent_size = 4 +[*.py] +indent_style = space +indent_size = 2 diff --git a/build.sh b/build.sh index 6158ce8..ab7ccd8 100755 --- a/build.sh +++ b/build.sh @@ -23,7 +23,7 @@ VERSION=`cat share/VERSION.txt` DOCKER_XTRA_ARGS="" function usage { - echo "Usage: $0 {test|dist|sign|clean|veryclean|docker [--args \"docker-args\"]|rat|githooks|docker-test}" + echo "Usage: $0 {lint|test|dist|sign|clean|veryclean|docker [--args \"docker-args\"]|rat|githooks|docker-test}" exit 1 } @@ -40,6 +40,12 @@ do shift case "$target" in + lint) + for lang_dir in lang/*; do + (cd "$lang_dir" && ./build.sh lint) + done + ;; + test) # run lang-specific tests (cd lang/java; ./build.sh test) @@ -50,8 +56,8 @@ do # install java artifacts required by other builds and interop tests mvn -B install -DskipTests - (cd lang/py; ./build.sh test) - (cd lang/py3; ./build.sh test) + (cd lang/py && ./build.sh lint test) + (cd lang/py3 && ./build.sh lint test) (cd lang/c; ./build.sh test) (cd lang/c++; ./build.sh test) (cd lang/csharp; ./build.sh test) diff --git a/lang/c++/build.sh b/lang/c++/build.sh index bd1a0b8..543638b 100755 --- a/lang/c++/build.sh +++ b/lang/c++/build.sh @@ -18,7 +18,7 @@ set -e # exit on error function usage { - echo "Usage: $0 {test|dist|clean|install|doc}" + echo "Usage: $0 {lint|test|dist|clean|install|doc}" exit 1 } @@ -76,6 +76,10 @@ for target in "$@" do case "$target" in + lint) + echo 'This is a stub where someone can provide linting.' + ;; + test) (cd build && cmake -G "Unix Makefiles" -D CMAKE_BUILD_TYPE=Debug -D AVRO_ADD_PROTECTOR_FLAGS=1 .. && make && cd .. \ && ./build/buffertest \ diff --git a/lang/c/build.sh b/lang/c/build.sh index a813b64..ff0c16f 100755 --- a/lang/c/build.sh +++ b/lang/c/build.sh @@ -19,7 +19,6 @@ # set -e # exit on error -#set -x root_dir=$(pwd) build_dir="../../build/c" @@ -57,6 +56,10 @@ case "$1" in $build_dir/tests/test_interop_data "../../build/interop/data" ;; + lint) + echo 'This is a stub where someone can provide linting.' + ;; + test) prepare_build make -C $build_dir @@ -88,7 +91,7 @@ case "$1" in ;; *) - echo "Usage: $0 {interop-data-generate|interop-data-test|test|dist|clean}" + echo "Usage: $0 {interop-data-generate|interop-data-test|lint|test|dist|clean}" exit 1 esac diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh index db344e2..afa4859 100755 --- a/lang/csharp/build.sh +++ b/lang/csharp/build.sh @@ -25,6 +25,10 @@ VERSION=`cat $ROOT/share/VERSION.txt` case "$1" in + lint) + echo 'This is a stub where someone can provide linting.' + ;; + test) dotnet build --configuration Release Avro.sln @@ -77,8 +81,6 @@ case "$1" in ;; *) - echo "Usage: $0 {test|clean|dist|perf|interop-data-generate|interop-data-test}" + echo "Usage: $0 {lint|test|clean|dist|perf|interop-data-generate|interop-data-test}" exit 1 esac - -exit 0 diff --git a/lang/java/build.sh b/lang/java/build.sh index f0f0f32..88847ea 100755 --- a/lang/java/build.sh +++ b/lang/java/build.sh @@ -15,51 +15,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e # exit on error +set -e -function usage { - echo "Usage: $0 {test|dist|clean}" +usage() { + echo "Usage: $0 {lint|test|dist|clean}" exit 1 } -if [ $# -eq 0 ] -then - usage -fi - -if [ -f VERSION.txt ] -then - VERSION=`cat VERSION.txt` -else - VERSION=`cat ../../share/VERSION.txt` -fi - -for target in "$@" -do - -function do_dist() { - mvn -P dist package -DskipTests javadoc:aggregate +main() { + local target + (( $# )) || usage + for target; do + case "$target" in + lint) + mvn -B spotless:apply + ;; + test) + mvn -B test + # Test the modules that depend on hadoop using Hadoop 3 + mvn -B test -Phadoop3 + ;; + dist) + mvn -P dist package -DskipTests javadoc:aggregate + ;; + clean) + mvn clean + ;; + *) + usage + ;; + esac + done } -case "$target" in - test) - mvn -B test - # Test the modules that depend on hadoop using Hadoop 3 - mvn -B test -Phadoop3 - ;; - - dist) - do_dist - ;; - - clean) - mvn clean - ;; - - *) - usage -esac - -done - -exit 0 +main "$@" diff --git a/lang/java/checkstyle.xml b/lang/java/checkstyle.xml index 0391815..e817cf8 100644 --- a/lang/java/checkstyle.xml +++ b/lang/java/checkstyle.xml @@ -27,8 +27,6 @@ <property name="file" value="${checkstyle.suppressions.file}"/> </module> - <module name="SuppressionCommentFilter"/> - <module name="FileTabCharacter"/> <module name="NewlineAtEndOfFile"> <property name="lineSeparator" value="lf"/> diff --git a/lang/js/build.sh b/lang/js/build.sh index deafeb5..a6cdb6b 100755 --- a/lang/js/build.sh +++ b/lang/js/build.sh @@ -20,6 +20,9 @@ set -e cd `dirname "$0"` case "$1" in + lint) + echo 'This is a stub where someone can provide linting.' + ;; test) npm install npm run cover @@ -33,8 +36,6 @@ case "$1" in rm -rf coverage ;; *) - echo "Usage: $0 {test|dist|clean}" >&2 + echo "Usage: $0 {lint|test|dist|clean}" >&2 exit 1 esac - -exit 0 diff --git a/lang/perl/build.sh b/lang/perl/build.sh index 95cc629..fee0ca6 100755 --- a/lang/perl/build.sh +++ b/lang/perl/build.sh @@ -18,7 +18,7 @@ set -e # exit on error function usage { - echo "Usage: $0 {test|dist|clean}" + echo "Usage: $0 {lint|test|dist|clean}" exit 1 } @@ -44,6 +44,9 @@ function do_clean(){ } case "$target" in + lint) + echo 'This is a stub where someone can provide linting.' + ;; test) perl ./Makefile.PL && make test ;; diff --git a/lang/php/build.sh b/lang/php/build.sh index 46a568c..796c41e 100755 --- a/lang/php/build.sh +++ b/lang/php/build.sh @@ -53,6 +53,10 @@ case "$1" in phpunit test/InterOpTest.php ;; + lint) + echo 'This is a stub where someone can provide linting.' + ;; + test) phpunit test/AllTests.php ;; @@ -66,7 +70,7 @@ case "$1" in ;; *) - echo "Usage: $0 {interop-data-generate|test-interop|test|dist|clean}" + echo "Usage: $0 {interop-data-generate|test-interop|lint|test|dist|clean}" esac diff --git a/lang/py/.gitignore b/lang/py/.gitignore index df3cd4d..44d9b58 100644 --- a/lang/py/.gitignore +++ b/lang/py/.gitignore @@ -1,3 +1,4 @@ build/ lib/ userlogs/ +*.egg-info/ diff --git a/lang/py/build.sh b/lang/py/build.sh index ff61989..662d854 100755 --- a/lang/py/build.sh +++ b/lang/py/build.sh @@ -15,46 +15,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e # exit on error +set -e -function usage { - echo "Usage: $0 {test|dist|clean}" +usage() { + echo "Usage: $0 {lint|test|dist|clean}" exit 1 } -if [ $# -eq 0 ] -then - usage -fi - -if [ -f VERSION.txt ] -then - VERSION=`cat VERSION.txt` -else - VERSION=`cat ../../share/VERSION.txt` -fi - -for target in "$@" -do - -case "$target" in - test) - ant test - ;; - - dist) - ant dist - ;; - - clean) - ant clean - rm -rf userlogs/ - ;; - - *) - usage -esac - -done +main() { + local target + (( $# )) || usage + for target; do + case "$target" in + lint) + ./setup.py isort + pycodestyle . + ;; + test) + ant test + ;; + dist) + ant dist + ;; + clean) + ant clean + rm -rf userlogs/ + ;; + *) + usage + ;; + esac + done +} -exit 0 +main "$@" diff --git a/lang/py3/build.sh b/lang/py/setup.cfg old mode 100755 new mode 100644 similarity index 65% copy from lang/py3/build.sh copy to lang/py/setup.cfg index 1bb27ad..dc07a1d --- a/lang/py3/build.sh +++ b/lang/py/setup.cfg @@ -1,5 +1,4 @@ -#!/bin/bash - +## # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -7,7 +6,7 @@ # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # -# https://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -15,25 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e # exit on error - -usage() { - echo "Usage: $0 {test|dist|clean}" -} - -main() { - local target - if (( $# == 0 )); then - usage - return 1 - fi - for target; do - case "$target" in - clean|dist|test) : ;; - *) usage; return 1 ;; - esac - done - python3 setup.py "$@" -} +[isort] +line_length = 150 +known_third_party=zope -main "$@" +[pycodestyle] +max-line-length = 150 +statistics = True +exclude = build +ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,W191,W291,W292,W293,W391,W503,W601 diff --git a/lang/py/src/avro/datafile.py b/lang/py/src/avro/datafile.py index c85a896..772d949 100644 --- a/lang/py/src/avro/datafile.py +++ b/lang/py/src/avro/datafile.py @@ -17,12 +17,13 @@ Read/Write Avro File Object Containers. """ import zlib + +from avro import io, schema + try: from cStringIO import StringIO except ImportError: from StringIO import StringIO -from avro import schema -from avro import io try: import snappy has_snappy = True diff --git a/lang/py/src/avro/io.py b/lang/py/src/avro/io.py index d94f0ff..c36c5b3 100644 --- a/lang/py/src/avro/io.py +++ b/lang/py/src/avro/io.py @@ -38,18 +38,14 @@ uses the following mapping: * Schema booleans are implemented as bool. """ +import datetime +import json import struct -from avro import schema -from avro import constants -from avro import timezones import sys from binascii import crc32 -import datetime +from decimal import Decimal, getcontext -from decimal import Decimal -from decimal import getcontext - -import json +from avro import constants, schema, timezones # # Constants diff --git a/lang/py/src/avro/ipc.py b/lang/py/src/avro/ipc.py index c616ac0..8cbf07b 100644 --- a/lang/py/src/avro/ipc.py +++ b/lang/py/src/avro/ipc.py @@ -17,13 +17,13 @@ Support for inter-process calls. """ import httplib + +from avro import io, protocol, schema + try: from cStringIO import StringIO except ImportError: from StringIO import StringIO -from avro import io -from avro import protocol -from avro import schema # # Constants @@ -482,4 +482,3 @@ class HTTPTransceiver(object): # # Server Implementations (none yet) # - diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py index 52684c2..e694d7e 100644 --- a/lang/py/src/avro/schema.py +++ b/lang/py/src/avro/schema.py @@ -35,10 +35,8 @@ A schema may be one of: Null. """ -from math import floor -from math import log10 - import json +from math import floor, log10 from avro import constants diff --git a/lang/py/src/avro/tether/tether_task.py b/lang/py/src/avro/tether/tether_task.py index 2588e1a..23112a7 100644 --- a/lang/py/src/avro/tether/tether_task.py +++ b/lang/py/src/avro/tether/tether_task.py @@ -18,19 +18,17 @@ __all__=["TetherTask","TaskType","inputProtocol","outputProtocol","HTTPRequestor"] -from avro import schema, protocol -from avro import io as avio -from avro import ipc - +import collections import io as pyio -import sys +import logging import os +import sys +import threading import traceback -import logging -import collections from StringIO import StringIO -import threading +from avro import io as avio +from avro import ipc, protocol, schema # create protocol objects for the input and output protocols # The build process should copy InputProtocol.avpr and OutputProtocol.avpr diff --git a/lang/py/src/avro/tether/tether_task_runner.py b/lang/py/src/avro/tether/tether_task_runner.py index 6cea49b..33d224a 100644 --- a/lang/py/src/avro/tether/tether_task_runner.py +++ b/lang/py/src/avro/tether/tether_task_runner.py @@ -26,13 +26,15 @@ if __name__ == "__main__": else: from . import TetherTask, find_port, inputProtocol -from avro import ipc -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer import logging -import weakref -import threading import sys +import threading import traceback +import weakref +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +from avro import ipc + class TaskRunnerResponder(ipc.Responder): """ diff --git a/lang/py/src/avro/tether/util.py b/lang/py/src/avro/tether/util.py index e223db3..cbeeef0 100644 --- a/lang/py/src/avro/tether/util.py +++ b/lang/py/src/avro/tether/util.py @@ -31,4 +31,4 @@ def find_port(): port=s.getsockname()[1] s.close() - return port \ No newline at end of file + return port diff --git a/lang/py/src/avro/timezones.py b/lang/py/src/avro/timezones.py index 7748f83..a4985b4 100644 --- a/lang/py/src/avro/timezones.py +++ b/lang/py/src/avro/timezones.py @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime -from datetime import timedelta -from datetime import tzinfo +from datetime import datetime, timedelta, tzinfo class UTCTzinfo(tzinfo): diff --git a/lang/py/src/avro/txipc.py b/lang/py/src/avro/txipc.py index a8572c6..72d63a6 100644 --- a/lang/py/src/avro/txipc.py +++ b/lang/py/src/avro/txipc.py @@ -19,17 +19,16 @@ try: from cStringIO import StringIO except ImportError: from StringIO import StringIO -from avro import ipc -from avro import io - from zope.interface import implements +from avro import io, ipc +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.protocol import Protocol +from twisted.web import resource, server from twisted.web.client import Agent from twisted.web.http_headers import Headers -from twisted.internet.defer import maybeDeferred, Deferred from twisted.web.iweb import IBodyProducer -from twisted.web import resource, server -from twisted.internet.protocol import Protocol + class TwistedRequestor(ipc.BaseRequestor): """A Twisted-compatible requestor. Returns a Deferred that will fire with the diff --git a/lang/py/test/av_bench.py b/lang/py/test/av_bench.py index 03ab8ad..e90c987 100644 --- a/lang/py/test/av_bench.py +++ b/lang/py/test/av_bench.py @@ -18,13 +18,12 @@ import sys import time -from random import sample, choice, randint +from random import choice, randint, sample from string import lowercase import avro.datafile -import avro.schema import avro.io - +import avro.schema types = ["A", "CNAME"] diff --git a/lang/py/test/gen_interop_data.py b/lang/py/test/gen_interop_data.py index d8cd857..336434e 100644 --- a/lang/py/test/gen_interop_data.py +++ b/lang/py/test/gen_interop_data.py @@ -16,9 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys -from avro import schema -from avro import io -from avro import datafile + +from avro import datafile, io, schema DATUM = { 'intField': 12, diff --git a/lang/py/test/mock_tether_parent.py b/lang/py/test/mock_tether_parent.py index 99f4153..c82e249 100644 --- a/lang/py/test/mock_tether_parent.py +++ b/lang/py/test/mock_tether_parent.py @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import socket import sys -import set_avro_test_path from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from avro import ipc -from avro import protocol -from avro import tether -import socket +import set_avro_test_path +from avro import ipc, protocol, tether + def find_port(): """ diff --git a/lang/py/test/sample_http_client.py b/lang/py/test/sample_http_client.py index 1ae29e4..4ad4925 100644 --- a/lang/py/test/sample_http_client.py +++ b/lang/py/test/sample_http_client.py @@ -17,8 +17,7 @@ # limitations under the License. import sys -from avro import ipc -from avro import protocol +from avro import ipc, protocol MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", diff --git a/lang/py/test/sample_http_server.py b/lang/py/test/sample_http_server.py index 9f05e94..e412ab5 100644 --- a/lang/py/test/sample_http_server.py +++ b/lang/py/test/sample_http_server.py @@ -15,9 +15,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from avro import ipc -from avro import protocol + +from avro import ipc, protocol MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", diff --git a/lang/py/test/set_avro_test_path.py b/lang/py/test/set_avro_test_path.py index 8c9e242..8e47faf 100644 --- a/lang/py/test/set_avro_test_path.py +++ b/lang/py/test/set_avro_test_path.py @@ -28,8 +28,8 @@ being built. To work around this the unittests import this module before importing AVRO. This module in turn adjusts the python path so that the test build of AVRO is higher on the path then any installed eggs. """ -import sys import os +import sys # determine the build directory and then make sure all paths that start with the # build directory are at the top of the path @@ -37,4 +37,4 @@ builddir=os.path.split(os.path.split(__file__)[0])[0] bpaths=filter(lambda s:s.startswith(builddir), sys.path) for p in bpaths: - sys.path.insert(0,p) \ No newline at end of file + sys.path.insert(0,p) diff --git a/lang/py/test/test_datafile.py b/lang/py/test/test_datafile.py index 102d9b0..2b7061c 100644 --- a/lang/py/test/test_datafile.py +++ b/lang/py/test/test_datafile.py @@ -17,10 +17,7 @@ import os import unittest import set_avro_test_path - -from avro import schema -from avro import io -from avro import datafile +from avro import datafile, io, schema SCHEMAS_TO_VALIDATE = ( ('"null"', None), diff --git a/lang/py/test/test_datafile_interop.py b/lang/py/test/test_datafile_interop.py index 545ce6c..ee02f99 100644 --- a/lang/py/test/test_datafile_interop.py +++ b/lang/py/test/test_datafile_interop.py @@ -17,9 +17,8 @@ import os import unittest import set_avro_test_path +from avro import datafile, io -from avro import io -from avro import datafile class TestDataFileInterop(unittest.TestCase): def test_interop(self): diff --git a/lang/py/test/test_io.py b/lang/py/test/test_io.py index 8ba0092..533aa40 100644 --- a/lang/py/test/test_io.py +++ b/lang/py/test/test_io.py @@ -13,22 +13,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import datetime import unittest - +from binascii import hexlify from decimal import Decimal +import set_avro_test_path +from avro import io, schema, timezones + try: from cStringIO import StringIO except ImportError: from StringIO import StringIO -from binascii import hexlify -import datetime -import set_avro_test_path -from avro import schema -from avro import io -from avro import timezones SCHEMAS_TO_VALIDATE = ( ('"null"', None), diff --git a/lang/py/test/test_ipc.py b/lang/py/test/test_ipc.py index 97d280e..575a0c9 100644 --- a/lang/py/test/test_ipc.py +++ b/lang/py/test/test_ipc.py @@ -20,11 +20,11 @@ servers yet available. import unittest import set_avro_test_path - # This test does import this code, to make sure it at least passes # compilation. from avro import ipc + class TestIPC(unittest.TestCase): def test_placeholder(self): pass diff --git a/lang/py/test/test_protocol.py b/lang/py/test/test_protocol.py index 483dfcf..3a8649f 100644 --- a/lang/py/test/test_protocol.py +++ b/lang/py/test/test_protocol.py @@ -17,8 +17,10 @@ Test the protocol parsing logic. """ import unittest + from avro import protocol + class ExampleProtocol(object): def __init__(self, protocol_string, valid, name='', comment=''): self._protocol_string = protocol_string diff --git a/lang/py/test/test_schema.py b/lang/py/test/test_schema.py index a4ae4cf..dd5aa8a 100644 --- a/lang/py/test/test_schema.py +++ b/lang/py/test/test_schema.py @@ -18,11 +18,10 @@ Test the schema parsing logic. """ import unittest -from avro.schema import SchemaParseException, AvroException - import set_avro_test_path - from avro import schema +from avro.schema import AvroException, SchemaParseException + def print_test_name(test_name): print '' diff --git a/lang/py/test/test_script.py b/lang/py/test/test_script.py index df02a98..214fc15 100644 --- a/lang/py/test/test_script.py +++ b/lang/py/test/test_script.py @@ -14,37 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import csv -from cStringIO import StringIO import json +import unittest +from cStringIO import StringIO +from operator import itemgetter +from os import remove +from os.path import dirname, isfile, join +from subprocess import check_call, check_output from tempfile import NamedTemporaryFile + import avro.schema -from avro.io import DatumWriter from avro.datafile import DataFileWriter -from os.path import dirname, join, isfile -from os import remove -from operator import itemgetter +from avro.io import DatumWriter NUM_RECORDS = 7 -try: - from subprocess import check_output -except ImportError: - from subprocess import Popen, PIPE - - def check_output(args): - pipe = Popen(args, stdout=PIPE) - if pipe.wait() != 0: - raise ValueError - return pipe.stdout.read() - -try: - from subprocess import check_call -except ImportError: - def check_call(args, **kw): - pipe = Popen(args, **kw) - assert pipe.wait() == 0 SCHEMA = ''' { diff --git a/lang/py/test/test_tether_task.py b/lang/py/test/test_tether_task.py index ed87083..9933070 100644 --- a/lang/py/test/test_tether_task.py +++ b/lang/py/test/test_tether_task.py @@ -24,6 +24,7 @@ import unittest import set_avro_test_path + class TestTetherTask(unittest.TestCase): """ TODO: We should validate the the server response by looking at stdout @@ -113,4 +114,4 @@ class TestTetherTask(unittest.TestCase): pass if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/lang/py/test/test_tether_word_count.py b/lang/py/test/test_tether_word_count.py index 3488462..cc34f2b 100644 --- a/lang/py/test/test_tether_word_count.py +++ b/lang/py/test/test_tether_word_count.py @@ -15,14 +15,15 @@ # limitations under the License. import inspect +import os import subprocess import sys import time import unittest -import os import set_avro_test_path + class TestTetherWordCount(unittest.TestCase): """ unittest for a python tethered map-reduce job. """ diff --git a/lang/py/test/txsample_http_client.py b/lang/py/test/txsample_http_client.py index 807e50f..3841062 100644 --- a/lang/py/test/txsample_http_client.py +++ b/lang/py/test/txsample_http_client.py @@ -17,12 +17,10 @@ # limitations under the License. import sys -from twisted.internet import reactor, defer +from avro import protocol, txipc +from twisted.internet import defer, reactor from twisted.python.util import println -from avro import protocol -from avro import txipc - MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", "protocol": "Mail", diff --git a/lang/py/test/txsample_http_server.py b/lang/py/test/txsample_http_server.py index 36cdb9f..604ef54 100644 --- a/lang/py/test/txsample_http_server.py +++ b/lang/py/test/txsample_http_server.py @@ -15,12 +15,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.web import server -from twisted.internet import reactor -from avro import ipc -from avro import protocol -from avro import txipc +from avro import ipc, protocol, txipc +from twisted.internet import reactor +from twisted.web import server MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", diff --git a/lang/py/test/word_count_task.py b/lang/py/test/word_count_task.py index ad911ad..24f2da2 100644 --- a/lang/py/test/word_count_task.py +++ b/lang/py/test/word_count_task.py @@ -18,9 +18,10 @@ __all__=["WordCountTask"] +import logging + from avro.tether import TetherTask -import logging #TODO::Make the logging level a parameter we can set #logging.basicConfig(level=logging.INFO) diff --git a/lang/py3/avro/datafile.py b/lang/py3/avro/datafile.py index adb2429..1e9b3fa 100644 --- a/lang/py3/avro/datafile.py +++ b/lang/py3/avro/datafile.py @@ -25,8 +25,8 @@ import logging import os import zlib -from avro import schema from avro import io as avro_io +from avro import schema try: import snappy diff --git a/lang/py3/avro/ipc.py b/lang/py3/avro/ipc.py index 0bca4cb..f5d376d 100644 --- a/lang/py3/avro/ipc.py +++ b/lang/py3/avro/ipc.py @@ -29,8 +29,7 @@ import os import socketserver from avro import io as avro_io -from avro import protocol -from avro import schema +from avro import protocol, schema logger = logging.getLogger(__name__) diff --git a/lang/py3/avro/protocol.py b/lang/py3/avro/protocol.py index b40fa08..30834da 100644 --- a/lang/py3/avro/protocol.py +++ b/lang/py3/avro/protocol.py @@ -21,11 +21,10 @@ Protocol implementation. """ -from types import MappingProxyType - import hashlib import json import logging +from types import MappingProxyType from avro import schema diff --git a/lang/py3/avro/schema.py b/lang/py3/avro/schema.py index b0b6c4c..9fbe8e5 100644 --- a/lang/py3/avro/schema.py +++ b/lang/py3/avro/schema.py @@ -38,12 +38,11 @@ A schema may be one of: - Null. """ -from types import MappingProxyType - import abc import json import logging import re +from types import MappingProxyType logger = logging.getLogger(__name__) diff --git a/lang/py3/avro/schemanormalization.py b/lang/py3/avro/schemanormalization.py index 9e6d364..88cddba 100644 --- a/lang/py3/avro/schemanormalization.py +++ b/lang/py3/avro/schemanormalization.py @@ -21,7 +21,7 @@ import hashlib from io import StringIO -from avro.schema import (ERROR, RECORD, UNION, ARRAY, MAP, ENUM, FIXED) +from avro.schema import ARRAY, ENUM, ERROR, FIXED, MAP, RECORD, UNION def ToParsingCanonicalForm(schema): diff --git a/lang/py3/avro/tests/av_bench.py b/lang/py3/avro/tests/av_bench.py index 78561c4..88cbafb 100644 --- a/lang/py3/avro/tests/av_bench.py +++ b/lang/py3/avro/tests/av_bench.py @@ -28,7 +28,6 @@ import avro.datafile import avro.io import avro.schema - TYPES = ('A', 'CNAME',) FILENAME = 'datafile.avr' diff --git a/lang/py3/avro/tests/gen_interop_data.py b/lang/py3/avro/tests/gen_interop_data.py index 4b83c30..c76f2c2 100644 --- a/lang/py3/avro/tests/gen_interop_data.py +++ b/lang/py3/avro/tests/gen_interop_data.py @@ -20,10 +20,7 @@ import sys -from avro import datafile -from avro import io -from avro import schema - +from avro import datafile, io, schema DATUM = { 'intField': 12, diff --git a/lang/py3/avro/tests/run_tests.py b/lang/py3/avro/tests/run_tests.py index 2111ec6..2575b36 100644 --- a/lang/py3/avro/tests/run_tests.py +++ b/lang/py3/avro/tests/run_tests.py @@ -49,12 +49,12 @@ import unittest from avro.tests.test_datafile import * from avro.tests.test_datafile_interop import * +from avro.tests.test_enum import * from avro.tests.test_io import * from avro.tests.test_ipc import * from avro.tests.test_protocol import * from avro.tests.test_schema import * from avro.tests.test_script import * -from avro.tests.test_enum import * def SetupLogging(): diff --git a/lang/py3/avro/tests/sample_http_client.py b/lang/py3/avro/tests/sample_http_client.py index 574d8c7..da8d030 100644 --- a/lang/py3/avro/tests/sample_http_client.py +++ b/lang/py3/avro/tests/sample_http_client.py @@ -19,8 +19,7 @@ # limitations under the License. import sys -from avro import ipc -from avro import protocol +from avro import ipc, protocol MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", diff --git a/lang/py3/avro/tests/sample_http_server.py b/lang/py3/avro/tests/sample_http_server.py index fd389d2..72518be 100644 --- a/lang/py3/avro/tests/sample_http_server.py +++ b/lang/py3/avro/tests/sample_http_server.py @@ -17,9 +17,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from avro import ipc -from avro import protocol + +from avro import ipc, protocol MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", diff --git a/lang/py3/avro/tests/test_datafile.py b/lang/py3/avro/tests/test_datafile.py index d5b07df..aa4042f 100644 --- a/lang/py3/avro/tests/test_datafile.py +++ b/lang/py3/avro/tests/test_datafile.py @@ -23,10 +23,18 @@ import os import tempfile import unittest -from avro import datafile -from avro import io -from avro import schema +from avro import datafile, io, schema +try: + import snappy + HAS_SNAPPY = True +except ImportError: + HAS_SNAPPY = False +try: + import zstandard + HAS_ZSTANDARD = True +except ImportError: + HAS_ZSTANDARD = False # ------------------------------------------------------------------------------ diff --git a/lang/py3/avro/tests/test_enum.py b/lang/py3/avro/tests/test_enum.py index 2db1f9e..040b3cb 100644 --- a/lang/py3/avro/tests/test_enum.py +++ b/lang/py3/avro/tests/test_enum.py @@ -22,6 +22,7 @@ import unittest from avro import schema + class TestEnum(unittest.TestCase): def testSymbolsInOrder(self): enum = schema.EnumSchema('Test', '', ['A', 'B'], schema.Names(), '', {}) diff --git a/lang/py3/avro/tests/test_io.py b/lang/py3/avro/tests/test_io.py index 6e44ee2..2fcd689 100644 --- a/lang/py3/avro/tests/test_io.py +++ b/lang/py3/avro/tests/test_io.py @@ -27,7 +27,6 @@ import unittest from avro import io as avro_io from avro import schema - SCHEMAS_TO_VALIDATE = ( ('"null"', None), ('"boolean"', True), diff --git a/lang/py3/avro/tests/test_ipc.py b/lang/py3/avro/tests/test_ipc.py index 225f4d0..20eee1a 100644 --- a/lang/py3/avro/tests/test_ipc.py +++ b/lang/py3/avro/tests/test_ipc.py @@ -27,9 +27,7 @@ import threading import time import unittest -from avro import ipc -from avro import protocol -from avro import schema +from avro import ipc, protocol, schema def NowMS(): diff --git a/lang/py3/avro/tests/test_normalization.py b/lang/py3/avro/tests/test_normalization.py index 8935529..148bfc0 100644 --- a/lang/py3/avro/tests/test_normalization.py +++ b/lang/py3/avro/tests/test_normalization.py @@ -21,7 +21,7 @@ import unittest from avro.schema import Parse -from avro.schemanormalization import ToParsingCanonicalForm, Fingerprint, FingerprintAlgorithmNames +from avro.schemanormalization import Fingerprint, FingerprintAlgorithmNames, ToParsingCanonicalForm class TestSchemaNormalization(unittest.TestCase): diff --git a/lang/py3/avro/tests/test_protocol.py b/lang/py3/avro/tests/test_protocol.py index 001db6b..abd17e8 100644 --- a/lang/py3/avro/tests/test_protocol.py +++ b/lang/py3/avro/tests/test_protocol.py @@ -27,7 +27,6 @@ import unittest from avro import protocol - # ------------------------------------------------------------------------------ diff --git a/lang/py3/avro/tests/test_schema.py b/lang/py3/avro/tests/test_schema.py index c2759da..2d82904 100644 --- a/lang/py3/avro/tests/test_schema.py +++ b/lang/py3/avro/tests/test_schema.py @@ -27,7 +27,6 @@ import unittest from avro import schema - # ------------------------------------------------------------------------------ diff --git a/lang/py3/avro/tests/test_script.py b/lang/py3/avro/tests/test_script.py index 1b267a1..647d346 100644 --- a/lang/py3/avro/tests/test_script.py +++ b/lang/py3/avro/tests/test_script.py @@ -32,7 +32,6 @@ import avro.datafile import avro.io import avro.schema - # ------------------------------------------------------------------------------ diff --git a/lang/py3/avro/tests/txsample_http_client.py b/lang/py3/avro/tests/txsample_http_client.py index 39c583a..fede960 100644 --- a/lang/py3/avro/tests/txsample_http_client.py +++ b/lang/py3/avro/tests/txsample_http_client.py @@ -19,12 +19,10 @@ # limitations under the License. import sys -from twisted.internet import reactor, defer +from avro import protocol, txipc +from twisted.internet import defer, reactor from twisted.python.util import println -from avro import protocol -from avro import txipc - MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", "protocol": "Mail", diff --git a/lang/py3/avro/tests/txsample_http_server.py b/lang/py3/avro/tests/txsample_http_server.py index 1936cc6..c85f42a 100644 --- a/lang/py3/avro/tests/txsample_http_server.py +++ b/lang/py3/avro/tests/txsample_http_server.py @@ -13,16 +13,15 @@ # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software + + +from avro import ipc, protocol, txipc # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +from twisted.internet import reactor # See the License for the specific language governing permissions and # limitations under the License. from twisted.web import server -from twisted.internet import reactor - -from avro import ipc -from avro import protocol -from avro import txipc MAIL_PROTOCOL_JSON = """\ {"namespace": "example.proto", diff --git a/lang/py3/avro/tool.py b/lang/py3/avro/tool.py index 28bf0dc..6bfce0d 100644 --- a/lang/py3/avro/tool.py +++ b/lang/py3/avro/tool.py @@ -25,13 +25,10 @@ NOTE: The API for the command-line tool is experimental. import sys import urllib +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from avro import datafile, io, ipc, protocol -from avro import io -from avro import datafile -from avro import protocol -from avro import ipc class GenericResponder(ipc.Responder): def __init__(self, proto, msg, datum): diff --git a/lang/py3/avro/txipc.py b/lang/py3/avro/txipc.py index 47a6bfe..d8e3225 100644 --- a/lang/py3/avro/txipc.py +++ b/lang/py3/avro/txipc.py @@ -20,12 +20,11 @@ import io -from avro import io as avro_io -from avro import ipc - from zope.interface import implements -from twisted.internet.defer import maybeDeferred, Deferred +from avro import io as avro_io +from avro import ipc +from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.protocol import Protocol from twisted.web import resource, server from twisted.web.client import Agent diff --git a/lang/py3/build.sh b/lang/py3/build.sh index 1bb27ad..fc4564f 100755 --- a/lang/py3/build.sh +++ b/lang/py3/build.sh @@ -18,7 +18,7 @@ set -e # exit on error usage() { - echo "Usage: $0 {test|dist|clean}" + echo "Usage: $0 {isort|lint|test|dist|clean}" } main() { @@ -29,8 +29,9 @@ main() { fi for target; do case "$target" in - clean|dist|test) : ;; - *) usage; return 1 ;; + lint) set -- isort "$@";; + clean|dist|isort|test) :;; + *) usage; return 1;; esac done python3 setup.py "$@" diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg index affeded..108c740 100644 --- a/lang/py3/setup.cfg +++ b/lang/py3/setup.cfg @@ -62,3 +62,12 @@ snappy = snappy [aliases] dist = sdist --dist-dir ../../dist/py3 + +[isort] +line_length = 150 +known_third_party=zope + +[pycodestyle] +max-line-length = 150 +statistics = True +ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,W503 diff --git a/lang/py3/setup.py b/lang/py3/setup.py index a752779..130ce18 100755 --- a/lang/py3/setup.py +++ b/lang/py3/setup.py @@ -27,13 +27,17 @@ https://pypi.org/project/avro-python3/ """ import distutils.command.clean -import distutils.file_util import distutils.dir_util +import distutils.errors +import distutils.file_util import distutils.log import fnmatch import os +import subprocess -from setuptools import setup +import setuptools + +import pycodestyle _HERE = os.path.dirname(os.path.abspath(__file__)) _AVRO_DIR = os.path.join(_HERE, 'avro') @@ -110,11 +114,53 @@ class CleanCommand(distutils.command.clean.clean): os.remove(name) +class GenerateInteropDataCommand(setuptools.Command): + """A command to generate Avro files for data interop test.""" + + user_options = [ + ('schema-file=', None, 'path to input Avro schema file'), + ('output-path=', None, 'path to output Avro data files'), + ] + + def initialize_options(self): + self.schema_file = os.path.join(os.getcwd(), 'interop.avsc') + self.output_path = os.getcwd() + + def finalize_options(self): + pass + + def run(self): + from avro.tests import gen_interop_data + gen_interop_data.generate(self.schema_file, self.output_path) + + +class LintCommand(setuptools.Command): + """Run pycodestyle on all your modules""" + description = __doc__ + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + subprocess.run(['pycodestyle', '.'], check=True) + except subprocess.CalledProcessError: + raise distutils.errors.DistutilsError("pycodestyle exited with a nonzero exit code.") + + def main(): if not _is_distribution(): _generate_package_data() - setup(cmdclass={"clean": CleanCommand}) + setuptools.setup(cmdclass={ + "clean": CleanCommand, + "generate_interop_data": GenerateInteropDataCommand, + "lint": LintCommand, + }) if __name__ == '__main__': diff --git a/lang/ruby/build.sh b/lang/ruby/build.sh index ebcd682..1f1319d 100755 --- a/lang/ruby/build.sh +++ b/lang/ruby/build.sh @@ -29,23 +29,24 @@ gem install --no-document -v 1.17.3 bundler bundle install case "$1" in - test) - bundle exec rake test - ;; - - dist) - bundle exec rake dist - ;; - - clean) - bundle exec rake clean - rm -rf tmp avro.gemspec data.avr - ;; - - *) - echo "Usage: $0 {test|dist|clean}" - exit 1 - + lint) + echo 'This is a stub where someone can provide linting.' + ;; + + test) + bundle exec rake test + ;; + + dist) + bundle exec rake dist + ;; + + clean) + bundle exec rake clean + rm -rf tmp avro.gemspec data.avr + ;; + + *) + echo "Usage: $0 {lint|test|dist|clean}" + exit 1 esac - -exit 0 diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile index 72a6811..c6944ce 100644 --- a/share/docker/Dockerfile +++ b/share/docker/Dockerfile @@ -53,6 +53,7 @@ RUN apt-get -qq update && \ g++ \ gcc \ git \ + isort \ libboost-all-dev \ libfontconfig1-dev \ libfreetype6-dev \ @@ -66,8 +67,15 @@ RUN apt-get -qq update && \ perl \ php5.6 \ php5.6-gmp \ + pycodestyle \ python \ + python-isort \ + python-pip \ python-setuptools \ + python-snappy \ + python-wheel \ + python3-isort \ + python3-pip \ python3-setuptools \ rake \ ruby \ diff --git a/share/docker/run-tests.sh b/share/docker/run-tests.sh index bf13887..a585a41 100755 --- a/share/docker/run-tests.sh +++ b/share/docker/run-tests.sh @@ -27,5 +27,5 @@ for lang in /avro/lang/*/ do headline "Run tests: $lang" cd "$lang" - ./build.sh test + ./build.sh lint test done
