LENS-1398: Python client not raising exception in some cases
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a3045fee Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a3045fee Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a3045fee Branch: refs/heads/current-release-line Commit: a3045fee12b3e750b6c931f5221a480d415efc76 Parents: bd344c7 Author: Rajat Khandelwal <pro...@apache.org> Authored: Tue Mar 21 16:09:05 2017 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Tue Mar 21 16:09:05 2017 +0530 ---------------------------------------------------------------------- contrib/clients/python/lens/client/main.py | 4 +- contrib/clients/python/lens/client/query.py | 43 ++++++++++++++++----- contrib/clients/python/setup.py | 2 + contrib/clients/python/test/test_lensclient.py | 13 +++++-- tools/conf/server/lens-site.xml | 6 +++ 5 files changed, 54 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/main.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/main.py b/contrib/clients/python/lens/client/main.py index bf5d81e..1e9d88f 100644 --- a/contrib/clients/python/lens/client/main.py +++ b/contrib/clients/python/lens/client/main.py @@ -21,7 +21,8 @@ from .log import LensLogClient from .session import LensSessionClient from .query import LensQueryClient from .utils import xml_file_to_conf - +import logging +logger = logging.getLogger(__name__) class LensClient(object): def __init__(self, base_url=None, username="", password="", database=None, conf=None): @@ -35,6 +36,7 @@ class LensClient(object): self.base_url = base_url or conf.get('lens.server.base.url', "http://0.0.0.0:9999/lensapi";) if self.base_url[-1] != '/': self.base_url += "/" + logger.debug("Using conf %s", conf) username = username or conf.get('lens.client.user.name', "anonymous") database = database or conf.get('lens.client.dbname') self.session = LensSessionClient(self.base_url, username, password, database, conf) http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/query.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/query.py b/contrib/clients/python/lens/client/query.py index f82f0cb..df16cca 100644 --- a/contrib/clients/python/lens/client/query.py +++ b/contrib/clients/python/lens/client/query.py @@ -15,21 +15,26 @@ # limitations under the License. # import codecs +import csv +import logging import time import zipfile import requests -from six import string_types, BytesIO, StringIO, PY2, PY3 +from requests.exceptions import HTTPError +from six import string_types, BytesIO, PY2, PY3 + from .models import WrappedJson from .utils import conf_to_xml -import csv +logger = logging.getLogger(__name__) long_type = int if PY3: from collections.abc import Iterable as Iterable elif PY2: from collections import Iterable as Iterable + long_type = long @@ -70,6 +75,7 @@ type_mappings = {'BOOLEAN': bool, } default_mapping = lambda x: x + class LensQueryResult(Iterable): def __init__(self, custom_mappings=None): if custom_mappings is None: @@ -93,6 +99,7 @@ class LensInMemoryResult(LensQueryResult): for row in self.rows: yield list(self._mapping(value.type)(value.value) if value else None for value in row['values']) + class LensPersistentResult(LensQueryResult): def __init__(self, header, response, encoding=None, is_header_present=True, delimiter=",", custom_mappings=None): @@ -107,6 +114,11 @@ class LensPersistentResult(LensQueryResult): def _parse_line(self, line): return list(self._mapping(self.header.columns[index].type)(line[index]) for index in range(len(line))) + def get_csv_reader(self, file): + if PY3: + file = codecs.iterdecode(file, 'utf-8') + return csv.reader(file, delimiter=self.delimiter) + def __iter__(self): if self.is_zipped: byte_stream = BytesIO(self.response.content) @@ -114,7 +126,7 @@ class LensPersistentResult(LensQueryResult): for name in self.zipfile.namelist(): with self.zipfile.open(name) as single_file: if name[-3:] == 'csv': - reader = csv.reader(single_file, delimiter=self.delimiter) + reader = self.get_csv_reader(single_file) else: reader = single_file reader_iterator = iter(reader) @@ -142,7 +154,8 @@ class LensQueryClient(object): self.launched_queries = [] self.finished_queries = {} self.query_confs = {} - self.is_header_present_in_result = self._session['lens.query.output.write.header'].lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup'] + self.is_header_present_in_result = self._session['lens.query.output.write.header'].lower() \ + in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup'] def __call__(self, **filters): filters['sessionid'] = self._session._sessionid @@ -183,6 +196,7 @@ class LensQueryClient(object): payload.append(('conf', conf_to_xml(conf))) resp = requests.post(self.base_url + "queries/", files=payload, headers={'accept': 'application/json'}) query = self.sanitize_response(resp) + logger.info("Submitted query %s", query) if conf: self.query_confs[str(query)] = conf if fetch_result: @@ -216,7 +230,8 @@ class LensQueryClient(object): return LensPersistentResult(metadata, resp, is_header_present=is_header_present, *args, **kwargs) else: response = requests.get(self.base_url + "queries/" + handle + "/resultset", - params={'sessionid': self._session._sessionid}, headers={'accept': 'application/json'}) + params={'sessionid': self._session._sessionid}, + headers={'accept': 'application/json'}) resp = self.sanitize_response(response) # If it has in memory result, return inmemory result iterator if resp._is_wrapper and resp._wrapped_key == u'inMemoryQueryResult': @@ -228,13 +243,18 @@ class LensQueryClient(object): raise Exception("Result set not available") def sanitize_response(self, resp): - resp.raise_for_status() try: resp_json = resp.json(object_hook=WrappedJson) + except: + resp_json = resp.json() + if resp_json is not None: if 'lensAPIResult' in resp_json: resp_json = resp_json.lens_a_p_i_result if 'error' in resp_json: - raise Exception(resp_json['error']) + error = resp_json['error'] + if "stackTrace" in error: + logger.error(error['stackTrace']) + raise HTTPError(error, request=resp.request, response=resp) if 'data' in resp_json: data = resp_json.data if len(data) == 2 and 'type' in data: @@ -242,6 +262,9 @@ class LensQueryClient(object): keys.remove('type') return WrappedJson({data['type']: data[keys[0]]}) return data - except: - resp_json = resp.json() - return resp_json + if resp_json is not None: + return resp_json + else: + resp.raise_for_status() + logger.error(resp.text) + raise Exception("Unknown error with response", resp) http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/setup.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/setup.py b/contrib/clients/python/setup.py index 113205d..de59d32 100644 --- a/contrib/clients/python/setup.py +++ b/contrib/clients/python/setup.py @@ -45,6 +45,8 @@ class Tox(TestCommand): def run_tests(self): #import here, cause outside the eggs aren't loaded import tox + import os + del os.environ["PYTHONPATH"] errcode = tox.cmdline(self.test_args) sys.exit(errcode) http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/test/test_lensclient.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/test/test_lensclient.py b/contrib/clients/python/test/test_lensclient.py index e9b0bdb..b8d7c73 100644 --- a/contrib/clients/python/test/test_lensclient.py +++ b/contrib/clients/python/test/test_lensclient.py @@ -29,7 +29,9 @@ from lens.client.models import WrappedJson from requests.exceptions import HTTPError from lens.client import LensClient - +import logging +logger = logging.getLogger(__name__) +logging.basicConfig(level="DEBUG") def check_output(command): output = subprocess.check_output(command.split()) @@ -85,11 +87,14 @@ class TestLensClient(object): cls.base_path = glob.glob(joined)[0] with cwd(cls.base_path): with cwd('server'): + logger.info("Inside server directory") server_start_output = check_output("bin/lens-ctl restart") assert "Started lens server" in server_start_output assert os.path.exists('logs/server.pid') + logger.info("started lens server") time.sleep(1) while not os.path.exists('logs/lensserver.log'): + logger.info("waiting for lensserver.log to be created") error = get_error() if has_error(error): # Assert again with complete error @@ -97,9 +102,11 @@ class TestLensClient(object): time.sleep(1) error = get_error() if has_error(error): + logger.error(error) assert False, error - + logger.info("finished setting up server environment. Will setup client now") with cwd('client'): + logger.info("Inside client directory") cls.candidate_query = select_query('examples/resources/cube-queries.sql') with open('check_connection.sql', 'w') as f: f.write('show databases') @@ -160,7 +167,7 @@ class TestLensClient(object): handle = client.queries.submit(self.candidate_query, query_name="Candidate Query") finished_query = client.queries.wait_till_finish(handle) assert client.queries[handle] == finished_query - queries = client.queries(state='SUCCESSFUL', fromDate=finished_query.submission_time - 1, + queries = client.queries(state='successful,failed', fromDate=finished_query.submission_time - 1, toDate=finished_query.submission_time + 1) assert handle in queries http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/tools/conf/server/lens-site.xml ---------------------------------------------------------------------- diff --git a/tools/conf/server/lens-site.xml b/tools/conf/server/lens-site.xml index b53ff25..c303198 100644 --- a/tools/conf/server/lens-site.xml +++ b/tools/conf/server/lens-site.xml @@ -36,4 +36,10 @@ <name>datanucleus.schema.autoCreateTables</name> <value>true</value> </property> + +<property> + <name>hive.metastore.schema.verification</name> + <value>false</value> +</property> + </configuration>
