This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix-queryserver.git
The following commit(s) were added to refs/heads/master by this push: new 1e78243 PHOENIX-5879 Fix recently introduced python 2.7 incompatibilities and flake8 warning 1e78243 is described below commit 1e78243c4611b4b0ac3f62f90685acf3959afbef Author: Istvan Toth <st...@apache.org> AuthorDate: Wed Apr 29 19:02:33 2020 +0200 PHOENIX-5879 Fix recently introduced python 2.7 incompatibilities and flake8 warning * fix time type bug introduced in previous patch * fix python27 incompatibilities * downgrade dependencies to make sure setup works on 2.7 * fix flake8 warnings additional improvements: * pare down dependencies * dockerize tox tests * update docs * expose ways to start a local test PQS server from maven * remove shell example, as it is python2 only * update author/contact info in setup.py * update docs * PHOENIX-5854 Synchronize the python client version to the queryserver version * set python 3 supported versions to 3.4-3.8 in docs, setup.py and tests * add simplified date/time/timestamp test * add unicode test --- python/README.md | 2 +- python/phoenixdb/Dockerfile | 9 ++ python/phoenixdb/Dockerfile-pqs | 9 ++ python/phoenixdb/NEWS.rst | 18 ++-- python/phoenixdb/README.rst | 74 ++++++-------- python/phoenixdb/docker-compose.yml | 21 ---- python/phoenixdb/examples/shell.py | 33 ------- python/phoenixdb/phoenixdb/__init__.py | 12 ++- python/phoenixdb/phoenixdb/avatica/client.py | 16 +-- python/phoenixdb/phoenixdb/connection.py | 5 +- python/phoenixdb/phoenixdb/cursor.py | 9 +- python/phoenixdb/phoenixdb/tests/__init__.py | 24 +++-- python/phoenixdb/phoenixdb/tests/test_avatica.py | 1 + python/phoenixdb/phoenixdb/tests/test_db.py | 8 +- python/phoenixdb/phoenixdb/tests/test_dbapi20.py | 5 +- python/phoenixdb/phoenixdb/tests/test_types.py | 34 ++++++- python/phoenixdb/phoenixdb/types.py | 107 +++++++++++---------- python/phoenixdb/requirements.txt | 4 - python/phoenixdb/setup.py | 22 ++++- python/phoenixdb/tox.ini | 7 +- .../phoenix/end2end/QueryServerBasicsIT.java | 7 +- .../end2end/SecureQueryServerPhoenixDBIT.java | 4 +- 22 files changed, 223 insertions(+), 208 deletions(-) diff --git a/python/README.md b/python/README.md index 45b4fc7..02eb3b9 100644 --- a/python/README.md +++ b/python/README.md @@ -23,7 +23,7 @@ This driver implements the Python DB 2.0 API for database drivers as described b This driver is implemented using the Phoenix Query Server (PQS) and the [Apache Calcite Avatica](https://calcite.apache.org/avatica) project. -This driver should be compatible with Python 2.7 and Python 3.3+ and support both unauthenticated access and +This driver should be compatible with Python 2.7 and Python 3.4+ and support both unauthenticated access and authenticated access via SPNEGO to PQS. ## Usage diff --git a/python/phoenixdb/Dockerfile b/python/phoenixdb/Dockerfile new file mode 100644 index 0000000..7edaced --- /dev/null +++ b/python/phoenixdb/Dockerfile @@ -0,0 +1,9 @@ +from themattrix/tox-base + +RUN apt-get update && apt-get install -y krb5-user libkrb5-dev + +ENV PHOENIXDB_TEST_DB_URL=http://host.docker.internal:8765 +ENV PHOENIXDB_TEST_DB_TRUSTSTORE $PHOENIXDB_TEST_DB_TRUSTSTORE +ENV PHOENIXDB_TEST_DB_AUTHENTICATION $PHOENIXDB_TEST_DB_AUTHENTICATION +ENV PHOENIXDB_TEST_DB_AVATICA_USER $PHOENIXDB_TEST_DB_AVATICA_USER +ENV PHOENIXDB_TEST_DB_AVATICA_PASSWORD $PHOENIXDB_TEST_DB_AVATICA_PASSWORD \ No newline at end of file diff --git a/python/phoenixdb/Dockerfile-pqs b/python/phoenixdb/Dockerfile-pqs new file mode 100644 index 0000000..7bfb7ca --- /dev/null +++ b/python/phoenixdb/Dockerfile-pqs @@ -0,0 +1,9 @@ +from maven:3-jdk-8 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq krb5-user libkrb5-dev + +EXPOSE 8765 + +# copy all the files to the container + +CMD mvn clean verify -am -pl queryserver-it -Dtest=foo -Dit.test=QueryServerBasicsIT#startLocalPQS -Ddo.not.randomize.pqs.port=true -Dstart.unsecure.pqs=true \ No newline at end of file diff --git a/python/phoenixdb/NEWS.rst b/python/phoenixdb/NEWS.rst index 3d4cff1..089e7ed 100644 --- a/python/phoenixdb/NEWS.rst +++ b/python/phoenixdb/NEWS.rst @@ -4,14 +4,16 @@ Changelog Unreleased ---------- - Replaced bundled requests_kerberos with request_gssapi library -- Refactor authentication code -- Support for specifying server certificate -- Support for BASIC and DIGEST authentication -- Fix HTTP error parsing -- Add transaction support -- Add list support -- Rewrite type handling -- Refactor test suite +- Refactored authentication code +- Added support for specifying server certificate +- Added support for BASIC and DIGEST authentication +- Fixed HTTP error parsing +- Added transaction support +- Added list support +- Rewritten type handling +- Refactored test suite +- Removed shell example, as it was python2 only +- Updated documentation Version 0.7 ----------- diff --git a/python/phoenixdb/README.rst b/python/phoenixdb/README.rst index 8388517..8f321af 100644 --- a/python/phoenixdb/README.rst +++ b/python/phoenixdb/README.rst @@ -1,14 +1,6 @@ Phoenix database adapter for Python =================================== -.. image:: https://code.oxygene.sk/lukas/python-phoenixdb/badges/master/pipeline.svg - :target: https://code.oxygene.sk/lukas/python-phoenixdb/commits/master - :alt: Build Status - -.. image:: https://readthedocs.org/projects/python-phoenixdb/badge/?version=latest - :target: http://python-phoenixdb.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - ``phoenixdb`` is a Python library for accessing the `Phoenix SQL database <http://phoenix.apache.org/>`_ using the @@ -20,14 +12,13 @@ which should be familiar to most Python programmers. Installation ------------ -The easiest way to install the library is using `pip <https://pip.pypa.io/en/stable/>`_:: - - pip install phoenixdb +The source code is part of the phoenix-queryserver source distribution. +You can download it from <https://phoenix.apache.org/>, or get the latest development version +from <https://github.com/apache/phoenix-queryserver> -You can also download the source code from `here <https://phoenix.apache.org/download.html>`_, -extract the archive and then install it manually:: +Extract the archive and then install it manually:: - cd /path/to/apache-phoenix-x.y.z/phoenix + cd /path/to/phoenix-queryserver-x.y.z/python/phoenixdb python setup.py install Usage @@ -65,17 +56,12 @@ necessary requirements:: pip install -r requirements.txt python setup.py develop -To create or update the Avatica protobuf classes, change the tag in ``gen-protobuf.sh`` -and run the script. - -If you need a Phoenix query server for experimenting, you can get one running -quickly using `Docker <https://www.docker.com/>`_:: - - docker-compose up - -Or if you need an older version of Phoenix:: +You can start a Phoenix QueryServer instance on http://localhost:8765 for testing by running +the following command in the phoenix-queryserver directory: - PHOENIX_VERSION=4.9 docker-compose up + mvn clean verify -am -pl queryserver-it -Dtest=foo \ + -Dit.test=QueryServerBasicsIT\#startLocalPQS \ + -Ddo.not.randomize.pqs.port=true -Dstart.unsecure.pqs=true If you want to use the library without installing the phoenixdb library, you can use the `PYTHONPATH` environment variable to point to the library directly:: @@ -85,24 +71,7 @@ the `PYTHONPATH` environment variable to point to the library directly:: cd ~/my_project PYTHONPATH=$PHOENIX_HOME/build/lib python my_app.py -Interactive SQL shell ---------------------- - -There is a Python-based interactive shell include in the examples folder, which can be -used to connect to Phoenix and execute queries:: - - ./examples/shell.py http://localhost:8765/ - db=> CREATE TABLE test (id INTEGER PRIMARY KEY, name VARCHAR); - no rows affected (1.363 seconds) - db=> UPSERT INTO test (id, name) VALUES (1, 'Lukas'); - 1 row affected (0.004 seconds) - db=> SELECT * FROM test; - +------+-------+ - | ID | NAME | - +======+=======+ - | 1 | Lukas | - +------+-------+ - 1 row selected (0.019 seconds) +Don't forget to run flake8 on your changes. Running the test suite ---------------------- @@ -114,6 +83,14 @@ working Phoenix database and set the ``PHOENIXDB_TEST_DB_URL`` environment varia export PHOENIXDB_TEST_DB_URL='http://localhost:8765/' nosetests +If you use a secure PQS server, you can set the connection parameters via the following environment +variables: + +- PHOENIXDB_TEST_DB_TRUSTSTORE +- PHOENIXDB_TEST_DB_AUTHENTICATION +- PHOENIXDB_TEST_DB_AVATICA_USER +- PHOENIXDB_TEST_DB_AVATICA_PASSWORD + Similarly, tox can be used to run the test suite against multiple Python versions:: pyenv install 3.5.5 @@ -122,6 +99,19 @@ Similarly, tox can be used to run the test suite against multiple Python version pyenv global 2.7.14 3.5.5 3.6.4 PHOENIXDB_TEST_DB_URL='http://localhost:8765' tox +You can use tox and docker to run the tests on all supported python versions without installing the +environments locally:: + + docker build -t toxtest . + docker run --rm -v `pwd`:/src toxtest + +You can also run the test suite from maven as part of the Java build by setting the +run.full.python.testsuite property. You DO NOT need to set the PHOENIXDB_* enviroment variables, +maven will set them up for you. The output of the test run will be saved in +phoenix-queryserver/queryserver-it/target/python-stdout.log and python-stderr.log:: + + mvn clean verify -Drun.full.python.testsuite=true + Known issues ------------ diff --git a/python/phoenixdb/docker-compose.yml b/python/phoenixdb/docker-compose.yml deleted file mode 100644 index bf398ec..0000000 --- a/python/phoenixdb/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - phoenix: - image: docker.oxygene.sk/lukas/python-phoenixdb/phoenix:${PHOENIX_VERSION:-4.11} - ports: - - "127.0.0.1:8765:8765" diff --git a/python/phoenixdb/examples/shell.py b/python/phoenixdb/examples/shell.py deleted file mode 100755 index 820435e..0000000 --- a/python/phoenixdb/examples/shell.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import argparse -import sqlline - -parser = argparse.ArgumentParser() -parser.add_argument('--debug', '-d', action='store_true') -parser.add_argument('url') -args = parser.parse_args() - -if args.debug: - logging.basicConfig(level=logging.DEBUG) - -with sqlline.SqlLine() as sqlline: - sqlline.connect('phoenixdb', args.url) - sqlline.connection.autocommit = True - sqlline.run() diff --git a/python/phoenixdb/phoenixdb/__init__.py b/python/phoenixdb/phoenixdb/__init__.py index 0a444e0..a90be99 100644 --- a/python/phoenixdb/phoenixdb/__init__.py +++ b/python/phoenixdb/phoenixdb/__init__.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from gssapi import mechs from phoenixdb import errors, types from phoenixdb.avatica import AvaticaClient @@ -19,10 +20,11 @@ from phoenixdb.connection import Connection from phoenixdb.errors import * # noqa: F401,F403 from phoenixdb.types import * # noqa: F401,F403 -from requests_gssapi import HTTPSPNEGOAuth, OPTIONAL -from gssapi import mechs; from requests.auth import HTTPBasicAuth, HTTPDigestAuth +from requests_gssapi import HTTPSPNEGOAuth, OPTIONAL + + __all__ = ['connect', 'apilevel', 'threadsafety', 'paramstyle'] + types.__all__ + errors.__all__ @@ -97,11 +99,11 @@ def connect(url, max_retries=None, auth=None, authentication=None, avatica_user= spnego = mechs.Mechanism.from_sasl_name("SPNEGO") if auth == "SPNEGO": - #Special case for backwards compatibility - auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego) + # Special case for backwards compatibility + auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech=spnego) elif auth is None and authentication is not None: if authentication == "SPNEGO": - auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego, opportunistic_auth=True) + auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech=spnego, opportunistic_auth=True) elif authentication == "BASIC" and avatica_user is not None and avatica_password is not None: auth = HTTPBasicAuth(avatica_user, avatica_password) elif authentication == "DIGEST" and avatica_user is not None and avatica_password is not None: diff --git a/python/phoenixdb/phoenixdb/avatica/client.py b/python/phoenixdb/phoenixdb/avatica/client.py index e392aee..daad12e 100644 --- a/python/phoenixdb/phoenixdb/avatica/client.py +++ b/python/phoenixdb/phoenixdb/avatica/client.py @@ -14,14 +14,14 @@ """Implementation of the JSON-over-HTTP RPC protocol used by Avatica.""" -import re -import socket -import pprint -import math import logging +import math +import pprint +import re import time + from phoenixdb import errors -from phoenixdb.avatica.proto import requests_pb2, common_pb2, responses_pb2 +from phoenixdb.avatica.proto import common_pb2, requests_pb2, responses_pb2 import requests @@ -165,13 +165,13 @@ class AvaticaClient(object): while True: logger.debug("POST %s %r %r", self.url.geturl(), body, headers) - requestArgs = {'data':body, 'stream':True, 'headers':headers} + requestArgs = {'data': body, 'stream': True, 'headers': headers} if self.auth is not None: - requestArgs.update(auth = self.auth) + requestArgs.update(auth=self.auth) if self.verify is not None: - requestArgs.update(verify = self.verify) + requestArgs.update(verify=self.verify) try: response = requests.request('post', self.url.geturl(), **requestArgs) diff --git a/python/phoenixdb/phoenixdb/connection.py b/python/phoenixdb/phoenixdb/connection.py index e041425..1cd562b 100644 --- a/python/phoenixdb/phoenixdb/connection.py +++ b/python/phoenixdb/phoenixdb/connection.py @@ -16,6 +16,7 @@ import logging import uuid import weakref + from phoenixdb import errors from phoenixdb.avatica.client import OPEN_CONNECTION_PROPERTIES from phoenixdb.cursor import Cursor @@ -99,12 +100,12 @@ class Connection(object): def commit(self): if self._closed: raise ProgrammingError('the connection is already closed') - self._client.commit(self._id); + self._client.commit(self._id) def rollback(self): if self._closed: raise ProgrammingError('the connection is already closed') - self._client.rollback(self._id); + self._client.rollback(self._id) def cursor(self, cursor_factory=None): """Creates a new cursor. diff --git a/python/phoenixdb/phoenixdb/cursor.py b/python/phoenixdb/phoenixdb/cursor.py index e1557c8..1dac835 100644 --- a/python/phoenixdb/phoenixdb/cursor.py +++ b/python/phoenixdb/phoenixdb/cursor.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import collections -from phoenixdb.types import TypeHelper -from phoenixdb.errors import ProgrammingError, InternalError +import logging + from phoenixdb.avatica.proto import common_pb2 +from phoenixdb.errors import InternalError, ProgrammingError +from phoenixdb.types import TypeHelper __all__ = ['Cursor', 'ColumnDescription', 'DictCursor'] @@ -182,7 +183,7 @@ class Cursor(object): else: typed_value.null = False if is_array: - if type(value) in [list,tuple]: + if type(value) in [list, tuple]: for element in value: if mutate_to is not None: element = mutate_to(element) diff --git a/python/phoenixdb/phoenixdb/tests/__init__.py b/python/phoenixdb/phoenixdb/tests/__init__.py index 44b62f7..b541929 100644 --- a/python/phoenixdb/phoenixdb/tests/__init__.py +++ b/python/phoenixdb/phoenixdb/tests/__init__.py @@ -15,11 +15,11 @@ import os import unittest + import phoenixdb -import time TEST_DB_URL = os.environ.get('PHOENIXDB_TEST_DB_URL') -#TEST_DB_URL = "http://localhost:8765" +# TEST_DB_URL = "http://localhost:8765" TEST_DB_TRUSTSTORE = os.environ.get('PHOENIXDB_TEST_DB_TRUSTSTORE') TEST_DB_AUTHENTICATION = os.environ.get('PHOENIXDB_TEST_DB_AUTHENTICATION') TEST_DB_AVATICA_USER = os.environ.get('PHOENIXDB_TEST_DB_AVATICA_USER') @@ -27,35 +27,39 @@ TEST_DB_AVATICA_PASSWORD = os.environ.get('PHOENIXDB_TEST_DB_AVATICA_PASSWORD') httpArgs = {} if TEST_DB_TRUSTSTORE is not None: - httpArgs.update(verify = TEST_DB_TRUSTSTORE) + httpArgs.update(verify=TEST_DB_TRUSTSTORE) if TEST_DB_AUTHENTICATION is not None: - httpArgs.update(authentication = TEST_DB_AUTHENTICATION) + httpArgs.update(authentication=TEST_DB_AUTHENTICATION) if TEST_DB_AVATICA_USER is not None: - httpArgs.update(avatica_user = TEST_DB_AVATICA_USER) + httpArgs.update(avatica_user=TEST_DB_AVATICA_USER) if TEST_DB_AVATICA_PASSWORD is not None: - httpArgs.update(avatica_password = TEST_DB_AVATICA_PASSWORD) + httpArgs.update(avatica_password=TEST_DB_AVATICA_PASSWORD) + @unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database") class DatabaseTestCase(unittest.TestCase): def setUp(self): self.conn = phoenixdb.connect(TEST_DB_URL, autocommit=True, **httpArgs) + def closeDb(): self.conn.close() self.addCleanup(closeDb) def reopen(self, **avaticaArgs): self.conn.close() - self.conn = phoenixdb.connect(TEST_DB_URL, **avaticaArgs, **httpArgs) + kwargs = avaticaArgs.copy() + kwargs.update(httpArgs) + self.conn = phoenixdb.connect(TEST_DB_URL, **kwargs) def addTableCleanup(self, name): def dropTable(): with self.conn.cursor() as cursor: - cursor.execute("DROP TABLE IF EXISTS {table}".format(table = name)) + cursor.execute("DROP TABLE IF EXISTS {table}".format(table=name)) self.addCleanup(dropTable) def createTable(self, name, statement): with self.conn.cursor() as cursor: - cursor.execute("DROP TABLE IF EXISTS {table}".format(table = name)) - cursor.execute(statement.format(table = name)) + cursor.execute("DROP TABLE IF EXISTS {table}".format(table=name)) + cursor.execute(statement.format(table=name)) self.addTableCleanup(name) diff --git a/python/phoenixdb/phoenixdb/tests/test_avatica.py b/python/phoenixdb/phoenixdb/tests/test_avatica.py index 6152814..20a7e0b 100644 --- a/python/phoenixdb/phoenixdb/tests/test_avatica.py +++ b/python/phoenixdb/phoenixdb/tests/test_avatica.py @@ -14,6 +14,7 @@ # limitations under the License. import unittest + from phoenixdb.avatica.client import parse_url, urlparse diff --git a/python/phoenixdb/phoenixdb/tests/test_db.py b/python/phoenixdb/phoenixdb/tests/test_db.py index 485e223..ea81e4a 100644 --- a/python/phoenixdb/phoenixdb/tests/test_db.py +++ b/python/phoenixdb/phoenixdb/tests/test_db.py @@ -14,10 +14,10 @@ # limitations under the License. import unittest + import phoenixdb.cursor from phoenixdb.errors import InternalError -from phoenixdb.tests import TEST_DB_URL -from phoenixdb.tests import DatabaseTestCase +from phoenixdb.tests import DatabaseTestCase, TEST_DB_URL @unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database") @@ -83,5 +83,5 @@ class PhoenixDatabaseTest(DatabaseTestCase): self.assertEqual(cursor.fetchall(), [[1, 'text 1']]) def test_transaction(self): - #Todo write some transaction tests - pass \ No newline at end of file + # Todo write some transaction tests + pass diff --git a/python/phoenixdb/phoenixdb/tests/test_dbapi20.py b/python/phoenixdb/phoenixdb/tests/test_dbapi20.py index 1be6cb0..c80ecb4 100644 --- a/python/phoenixdb/phoenixdb/tests/test_dbapi20.py +++ b/python/phoenixdb/phoenixdb/tests/test_dbapi20.py @@ -14,10 +14,11 @@ # limitations under the License. import unittest + import phoenixdb +from phoenixdb.tests import TEST_DB_URL, httpArgs + from . import dbapi20 -from phoenixdb.tests import TEST_DB_URL -from phoenixdb.tests import httpArgs @unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database") diff --git a/python/phoenixdb/phoenixdb/tests/test_types.py b/python/phoenixdb/phoenixdb/tests/test_types.py index 598b5b8..19c730d 100644 --- a/python/phoenixdb/phoenixdb/tests/test_types.py +++ b/python/phoenixdb/phoenixdb/tests/test_types.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import sys import unittest -import datetime -import phoenixdb from decimal import Decimal + +import phoenixdb from phoenixdb.tests import DatabaseTestCase @@ -226,6 +227,16 @@ class TypesTest(DatabaseTestCase): [5, None], ]) + # Minimal date/time/timestamp type test that doesn't trigger PHOENIX-4664 + def test_time_minimal(self): + self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val1 date, val2 time, val3 timestamp)") + with self.conn.cursor() as cursor: + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12', '2015-07-12 13:01:02', '2015-07-12 13:01:02.123')") + cursor.execute("SELECT * FROM phoenixdb_test_tbl1 ORDER BY id") + self.assertEqual(cursor.fetchall(), [ + [1, datetime.date(2015, 7, 12), datetime.time(13, 1, 2), datetime.datetime(2015, 7, 12, 13, 1, 2, 123000)] + ]) + @unittest.skip("https://issues.apache.org/jira/browse/CALCITE-796") def test_timestamp_full(self): self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val timestamp)") @@ -248,6 +259,25 @@ class TypesTest(DatabaseTestCase): cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id") self.assertEqual(cursor.fetchall(), [[1, 'abc'], [2, None], [3, 'abc'], [4, None], [5, None], [6, None]]) + @unittest.skipIf(sys.version_info[0] < 3, "phoenixdb doesn't support unicode strings in Python2") + def test_unicode(self): + self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar)") + with self.conn.cursor() as cursor: + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, \ + '\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p')") + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, '\u265E')") + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (3, '\U0001F600')") + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (4, ?)", + ['\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p']) + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (5, ?)", ['\u265E']) + cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (6, ?)", ['\U0001F600']) + cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id") + self.assertEqual(cursor.fetchall(), + [[1, '\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p'], + [2, '\u265E'], [3, '\U0001F600'], + [4, '\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p'], + [5, '\u265E'], [6, '\U0001F600']]) + def test_varchar_very_long(self): self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar)") with self.conn.cursor() as cursor: diff --git a/python/phoenixdb/phoenixdb/types.py b/python/phoenixdb/phoenixdb/types.py index 20f8572..e446eca 100644 --- a/python/phoenixdb/phoenixdb/types.py +++ b/python/phoenixdb/phoenixdb/types.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import sys import time -import datetime from decimal import Decimal + from phoenixdb.avatica.proto import common_pb2 -from builtins import staticmethod + __all__ = [ 'Date', 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks', @@ -90,7 +91,8 @@ def datetime_to_java_sql_timestamp(d): td = d - datetime.datetime(1970, 1, 1) return td.microseconds // 1000 + (td.seconds + td.days * 24 * 3600) * 1000 -#FIXME This doesn't seem to be used anywhere in the code + +# FIXME This doesn't seem to be used anywhere in the code class ColumnType(object): def __init__(self, eq_types): @@ -181,67 +183,70 @@ This mapping should be structured as: JDBC_TO_REP = dict([ # These are the standard types that are used in Phoenix - (-6, common_pb2.BYTE), #TINYINT - (5, common_pb2.SHORT), #SMALLINT - (4, common_pb2.INTEGER), #INTEGER - (-5, common_pb2.LONG), #BIGINT - (6, common_pb2.DOUBLE), #FLOAT - (8, common_pb2.DOUBLE), #DOUBLE - (2, common_pb2.BIG_DECIMAL), #NUMERIC - (1, common_pb2.STRING), #CHAR - (91, common_pb2.JAVA_SQL_DATE), #DATE - (93, common_pb2.JAVA_SQL_TIMESTAMP), #TIME - (-2, common_pb2.BYTE_STRING), #BINARY - (-3, common_pb2.BYTE_STRING), #VARBINARY - (16, common_pb2.BOOLEAN), #BOOLEAN + (-6, common_pb2.BYTE), # TINYINT + (5, common_pb2.SHORT), # SMALLINT + (4, common_pb2.INTEGER), # INTEGER + (-5, common_pb2.LONG), # BIGINT + (6, common_pb2.DOUBLE), # FLOAT + (8, common_pb2.DOUBLE), # DOUBLE + (2, common_pb2.BIG_DECIMAL), # NUMERIC + (1, common_pb2.STRING), # CHAR + (91, common_pb2.JAVA_SQL_DATE), # DATE + (92, common_pb2.JAVA_SQL_TIME), # TIME + (93, common_pb2.JAVA_SQL_TIMESTAMP), # TIMESTAMP + (-2, common_pb2.BYTE_STRING), # BINARY + (-3, common_pb2.BYTE_STRING), # VARBINARY + (16, common_pb2.BOOLEAN), # BOOLEAN # These are the Non-standard types defined by Phoenix - (19, common_pb2.JAVA_SQL_DATE), #UNSIGNED_DATE - (15, common_pb2.DOUBLE), #UNSIGNED_DOUBLE - (14, common_pb2.DOUBLE), #UNSIGNED_FLOAT - (9, common_pb2.INTEGER), #UNSIGNED_INT - (10, common_pb2.LONG), #UNSIGNED_LONG - (13, common_pb2.SHORT), #UNSIGNED_SMALLINT - (20, common_pb2.JAVA_SQL_TIMESTAMP), #UNSIGNED_TIMESTAMP - (11, common_pb2.BYTE), #UNSIGNED_TINYINT + (19, common_pb2.JAVA_SQL_DATE), # UNSIGNED_DATE + (15, common_pb2.DOUBLE), # UNSIGNED_DOUBLE + (14, common_pb2.DOUBLE), # UNSIGNED_FLOAT + (9, common_pb2.INTEGER), # UNSIGNED_INT + (10, common_pb2.LONG), # UNSIGNED_LONG + (13, common_pb2.SHORT), # UNSIGNED_SMALLINT + (20, common_pb2.JAVA_SQL_TIMESTAMP), # UNSIGNED_TIMESTAMP + (11, common_pb2.BYTE), # UNSIGNED_TINYINT # The following are not used by Phoenix, but some of these are used by Avaticafor # parameter types - (-7, common_pb2.BOOLEAN), #BIT - (7, common_pb2.DOUBLE), #REAL - (3, common_pb2.BIG_DECIMAL), #DECIMAL - (12, common_pb2.STRING), #VARCHAR - (-1, common_pb2.STRING), #LONGVARCHAR - (-4, common_pb2.BYTE_STRING), #LONGVARBINARY - (2004, common_pb2.BYTE_STRING), #BLOB - (2005, common_pb2.STRING), #CLOB - (-15, common_pb2.STRING), #NCHAR - (-9, common_pb2.STRING), #NVARCHAR - (-16, common_pb2.STRING), #LONGNVARCHAR - (2011, common_pb2.STRING), #NCLOB - (2009, common_pb2.STRING), #SQLXML + (-7, common_pb2.BOOLEAN), # BIT + (7, common_pb2.DOUBLE), # REAL + (3, common_pb2.BIG_DECIMAL), # DECIMAL + (12, common_pb2.STRING), # VARCHAR + (-1, common_pb2.STRING), # LONGVARCHAR + (-4, common_pb2.BYTE_STRING), # LONGVARBINARY + (2004, common_pb2.BYTE_STRING), # BLOB + (2005, common_pb2.STRING), # CLOB + (-15, common_pb2.STRING), # NCHAR + (-9, common_pb2.STRING), # NVARCHAR + (-16, common_pb2.STRING), # LONGNVARCHAR + (2011, common_pb2.STRING), # NCLOB + (2009, common_pb2.STRING), # SQLXML # These are defined by JDBC, but cannot be mapped - #NULL - #OTHER - #JAVA_OBJECT - #DISTINCT - #STRUCT - #ARRAY 2003 - We are handling this as a special case - #REF - #DATALINK - #ROWID - #REF_CURSOR - #TIME WITH TIMEZONE - #TIMESTAMP WITH TIMEZONE + # NULL + # OTHER + # JAVA_OBJECT + # DISTINCT + # STRUCT + # ARRAY 2003 - We are handling this as a special case + # REF + # DATALINK + # ROWID + # REF_CURSOR + # TIME WITH TIMEZONE + # TIMESTAMP WITH TIMEZONE ]) """Maps the JDBC Type IDs to Protobuf Reps """ JDBC_MAP = {} -for k,v in JDBC_TO_REP.items(): +for k, v in JDBC_TO_REP.items(): JDBC_MAP[k & 0xffffffff] = REP_MAP[v] """Flips the available types to allow for faster lookup by JDBC type ID It has the same format as REP_MAP, but is keyed by JDBC type ID """ + + class TypeHelper(object): @staticmethod @@ -293,7 +298,7 @@ class TypeHelper(object): @staticmethod def _from_jdbc(jdbc_code): if jdbc_code not in JDBC_MAP: - #This should not happen. It's either a bug, or Avatica has added new types + # This should not happen. It's either a bug, or Avatica has added new types raise NotImplementedError('JDBC TYPE CODE {} is not supported'.format(jdbc_code)) return JDBC_MAP[jdbc_code] diff --git a/python/phoenixdb/requirements.txt b/python/phoenixdb/requirements.txt index 72fd653..b061374 100644 --- a/python/phoenixdb/requirements.txt +++ b/python/phoenixdb/requirements.txt @@ -16,10 +16,6 @@ # specific language governing permissions and limitations # under the License. # --e git+https://bitbucket.org/lalinsky/python-sqlline.git#egg=sqlline -nose protobuf>=3.0.0 -sphinx -flake8 requests requests-gssapi diff --git a/python/phoenixdb/setup.py b/python/phoenixdb/setup.py index 3accb45..d1f0ce5 100644 --- a/python/phoenixdb/setup.py +++ b/python/phoenixdb/setup.py @@ -32,16 +32,16 @@ def readme(): return f.read() -version = "0.7" +version = "1.0.0.dev" setup( name="phoenixdb", version=version, description="Phoenix database adapter for Python", long_description=readme(), - author="Lukas Lalinsky", - author_email="lu...@oxygene.sk", - url="https://bitbucket.org/lalinsky/python-phoenixdb", + author="Apache Software Foundation", + author_email="d...@phoenix.apache.org", + url="http://phoenix.apache.org/python.html", license="Apache 2", packages=find_packages(), include_package_data=True, @@ -60,10 +60,24 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], install_requires=[ 'protobuf>=3.0.0', 'requests', 'requests-gssapi' + ], + tests_require=[ + 'nose', + 'flake8' + ], + setup_requires=[ + # Later versions don't work with python2.7 + 'Sphinx<2.0.0', + # These are Sphinx dependencies, included only to be version managed for python2 + 'MarkupSafe<2.0.0', + 'Jinja2<3.0.0', + 'pyparsing<3.0.0' ] ) diff --git a/python/phoenixdb/tox.ini b/python/phoenixdb/tox.ini index 908696a..49d7c06 100644 --- a/python/phoenixdb/tox.ini +++ b/python/phoenixdb/tox.ini @@ -14,11 +14,12 @@ # limitations under the License. [tox] -envlist = py27,py35,py36 - +envlist = py27,py34,py35,py36,py37,py38 [testenv] passenv = PHOENIXDB_TEST_DB_URL commands = - flake8 + flake8 phoenixdb nosetests -v deps = -rrequirements.txt + nose + flake8 diff --git a/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java index 76c38ee..0540a3d 100644 --- a/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java +++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java @@ -43,6 +43,7 @@ import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.queryserver.QueryServerProperties; import org.apache.phoenix.util.ThinClientUtil; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Rule; @@ -68,7 +69,9 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT { @BeforeClass public static synchronized void beforeClass() throws Exception { CONF = getTestClusterConfig(); - CONF.setInt(QueryServerProperties.QUERY_SERVER_HTTP_PORT_ATTRIB, 0); + if(System.getProperty("do.not.randomize.pqs.port") == null) { + CONF.setInt(QueryServerProperties.QUERY_SERVER_HTTP_PORT_ATTRIB, 0); + } String url = getUrl(); AVATICA_SERVER = new QueryServerThread(new String[] { url }, CONF, QueryServerBasicsIT.class.getName()); @@ -376,10 +379,10 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT { } } - @Ignore @Test //Quick and dirty way start up a local Phoenix+PQS instance for testing against public void startLocalPQS() throws Exception { + Assume.assumeNotNull(System.getProperty("start.unsecure.pqs")); System.out.println("CONN STRING:" + CONN_STRING); System.out.println("Tests suspended!!!"); System.out.println("Kill maven run to stop server"); diff --git a/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java index 40c5e03..95c32f5 100644 --- a/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java +++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java @@ -333,14 +333,13 @@ public class SecureQueryServerPhoenixDBIT { } //This takes about 300s, so we are not running this by default - @Ignore @Test public void testFullSuite() throws Exception { + Assume.assumeNotNull(System.getProperty("run.full.python.testsuite")); File file = new File("."); runShellScript("python", "-m", "unittest", "discover", "-v", "-s", Paths.get(file.getAbsolutePath(), "..","python", "phoenixdb").toString()); } - @Ignore @Test //Quick and dirty way start up a local Phoenix+PQS instance for testing against //When started, this will write a setup script into target/krb_setup.sh @@ -348,6 +347,7 @@ public class SecureQueryServerPhoenixDBIT { //and set the environment so that the python unit tests can run against this instance. //You'll need to kill the test manually public void startLocalPQS() throws Exception { + Assume.assumeNotNull(System.getProperty("start.secure.pqs")); runShellScript("sleep", "86400"); }