LENS-1011: Implement python client for Lens
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/12b95da0 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/12b95da0 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/12b95da0 Branch: refs/heads/current-release-line Commit: 12b95da07df3957c851dd8af7d6d19b36368134f Parents: 01a561c Author: raju <[email protected]> Authored: Fri Jun 17 15:46:03 2016 +0800 Committer: raju <[email protected]> Committed: Fri Jun 17 15:46:03 2016 +0800 ---------------------------------------------------------------------- contrib/clients/pom.xml | 37 +++++ contrib/clients/python/.gitignore | 73 +++++++++ contrib/clients/python/README | 20 +++ contrib/clients/python/lens/__init__.py | 16 ++ contrib/clients/python/lens/client/__init__.py | 20 +++ contrib/clients/python/lens/client/log.py | 25 +++ contrib/clients/python/lens/client/main.py | 61 ++++++++ contrib/clients/python/lens/client/models.py | 64 ++++++++ contrib/clients/python/lens/client/query.py | 89 +++++++++++ contrib/clients/python/lens/client/utils.py | 62 ++++++++ contrib/clients/python/pom.xml | 110 +++++++++++++ contrib/clients/python/setup.cfg | 19 +++ contrib/clients/python/setup.py | 82 ++++++++++ contrib/clients/python/test/test_lensclient.py | 161 ++++++++++++++++++++ contrib/clients/python/tox.ini | 53 +++++++ contrib/pom.xml | 37 +++++ pom.xml | 5 + 17 files changed, 934 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/clients/pom.xml b/contrib/clients/pom.xml new file mode 100644 index 0000000..42a7fb0 --- /dev/null +++ b/contrib/clients/pom.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <name>Lens Contributed Clients</name> + <parent> + <artifactId>lens-contrib</artifactId> + <groupId>org.apache.lens</groupId> + <version>2.6.0-beta-SNAPSHOT</version> + </parent> + + <artifactId>lens-contrib-clients</artifactId> + <packaging>pom</packaging> + <description>Lens Contrib clients jar</description> + <modules> + <module>python</module> + </modules> +</project> http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/.gitignore ---------------------------------------------------------------------- diff --git a/contrib/clients/python/.gitignore b/contrib/clients/python/.gitignore new file mode 100644 index 0000000..c4a5755 --- /dev/null +++ b/contrib/clients/python/.gitignore @@ -0,0 +1,73 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +build/ +dist/ +docs/_build/ +pip.egg-info/ +MANIFEST +.tox +*.egg +*.py[cod] +*~ +.coverage +coverage.xml + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + + +# Intellij +.idea/ http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/README ---------------------------------------------------------------------- diff --git a/contrib/clients/python/README b/contrib/clients/python/README new file mode 100644 index 0000000..732d3c9 --- /dev/null +++ b/contrib/clients/python/README @@ -0,0 +1,20 @@ +# lens-python-client +Lens Python Client + +## Installation +You can install like this: + + pip install ... + + +## Usage + if __name__ == '__main__': + with LensClient("http://lens.server.url/";, "user.name", database="db") as client: + handle = client.queries.submit("cube select ...", query_name="My first query") + # Optionally wait for completion. + while not client.queries[handle].finished: + time.sleep(20) # sleep 20 seconds + print client.queries[handle].result_set_path + # listing queries: + for handle in client.queries(state='RUNNING'): + print client.queries[handle] http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/__init__.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/__init__.py b/contrib/clients/python/lens/__init__.py new file mode 100644 index 0000000..cce3aca --- /dev/null +++ b/contrib/clients/python/lens/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/client/__init__.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/__init__.py b/contrib/clients/python/lens/client/__init__.py new file mode 100644 index 0000000..d59c5e9 --- /dev/null +++ b/contrib/clients/python/lens/client/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .main import LensClient + +__all__ = ['LensClient'] +__version__ = "2.6.0" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/client/log.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/log.py b/contrib/clients/python/lens/client/log.py new file mode 100644 index 0000000..a10798f --- /dev/null +++ b/contrib/clients/python/lens/client/log.py @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import requests + + +class LensLogClient(object): + def __init__(self, base_url): + self.base_url = base_url + "logs/" + + def __getitem__(self, item): + return requests.get(self.base_url + str(item)).text http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/client/main.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/main.py b/contrib/clients/python/lens/client/main.py new file mode 100644 index 0000000..dc60497 --- /dev/null +++ b/contrib/clients/python/lens/client/main.py @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +import requests +from six import string_types +from .log import LensLogClient +from .query import LensQueryClient +from .utils import conf_to_xml, xml_file_to_conf + + +class LensClient(object): + def __init__(self, base_url=None, username="", password="", database=None, conf=None): + if conf and isinstance(conf, string_types) and os.path.exists(conf): + if os.path.isdir(conf): + conf = os.path.join(conf, 'lens-client-site.xml') + if os.path.exists(conf): + conf = xml_file_to_conf(conf) + if not conf: + conf = {} + self.base_url = base_url or conf.get('lens.server.base.url', "http://0.0.0.0:9999/lensapi";) + if self.base_url[-1] != '/': + self.base_url += "/" + username = username or conf.get('lens.client.user.name', "anonymous") + database = database or conf.get('lens.client.dbname') + self.open_session(username, password, database, conf) + self.queries = LensQueryClient(self.base_url, self._sessionid) + self.logs = LensLogClient(self.base_url) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close_session() + + def close_session(self): + if self._sessionid: + requests.delete(self.base_url + "session/", params={'sessionid': self._sessionid}) + self._sessionid = None + + def open_session(self, username, password, database, conf): + payload = [('username', username), ('password', password), ('sessionconf', conf_to_xml(conf))] + if database: + payload.append(('database', database)) + r = requests.post(self.base_url + "session/", files=payload, headers={'accept': 'application/xml'}) + r.raise_for_status() + self._sessionid = r.text http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/client/models.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/models.py b/contrib/clients/python/lens/client/models.py new file mode 100644 index 0000000..7451fe9 --- /dev/null +++ b/contrib/clients/python/lens/client/models.py @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json + +from .utils import to_camel_case + + +class WrappedJson(dict): + _is_wrapper = False + + def __init__(self, seq=None, **kwargs): + super(WrappedJson, self).__init__(seq, **kwargs) + if len(self) == 1: + (self._wrapped_key, self._wrapped_value), = self.items() + self._is_wrapper = True + + def __getitem__(self, item): + for name in {item, to_camel_case(item)}: + if name in self: + return super(WrappedJson, self).__getitem__(name) + if self._is_wrapper: + if name in self._wrapped_value: + return self._wrapped_value[name] + raise Exception("Couldn't access " + str(item) + " in " + str(self)) + + def __getattr__(self, item): + try: + return self.__getitem__(item) + except: + pass + + def __str__(self): + if self._is_wrapper: + return str(self._wrapped_value) + return json.dumps(self, indent=2) + + def __repr__(self): + if self._is_wrapper: + return str(self._wrapped_value) + return super(WrappedJson, self).__repr__() + + def __eq__(self, other): + return super(WrappedJson, self).__eq__(other) or ( + self._is_wrapper and other._is_wrapper and str(self) == str(other)) + + +class LensQuery(WrappedJson): + @property + def finished(self): + return self.status.status in ('SUCCESSFUL', 'FAILED', 'CANCELED', 'CLOSED') http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/client/query.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/query.py b/contrib/clients/python/lens/client/query.py new file mode 100644 index 0000000..88ce719 --- /dev/null +++ b/contrib/clients/python/lens/client/query.py @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import requests +import time +from six import string_types +from .models import WrappedJson, LensQuery +from .utils import conf_to_xml + + +class LensQueryClient(object): + def __init__(self, base_url, sessionid): + self._sessionid = sessionid + self.base_url = base_url + "queryapi/" + self.launched_queries = [] + self.finished_queries = {} + + def __call__(self, **filters): + filters['sessionid'] = self._sessionid + resp = requests.get(self.base_url + "queries/", params=filters, headers={'accept': 'application/json'}) + return self.sanitize_response(resp) + + def __getitem__(self, item): + if isinstance(item, string_types): + if item in self.finished_queries: + return self.finished_queries[item] + resp = requests.get(self.base_url + "queries/" + item, params={'sessionid': self._sessionid}, + headers={'accept': 'application/json'}) + resp.raise_for_status() + query = LensQuery(resp.json(object_hook=WrappedJson)) + if query.finished: + self.finished_queries[item] = query + return query + elif isinstance(item, WrappedJson): + if item._is_wrapper: + return self[item._wrapped_value] + raise Exception("Can't get query: " + str(item)) + + def submit(self, query, operation="execute", query_name=None, timeout=None, conf=None): + payload = [('sessionid', self._sessionid), ('query', query), ('operation', operation)] + if query_name: + payload.append(('queryName', query_name)) + if timeout: + payload.append(('timeoutmillis', timeout)) + payload.append(('conf', conf_to_xml(conf))) + resp = requests.post(self.base_url + "queries/", files=payload, headers={'accept': 'application/json'}) + query = self.sanitize_response(resp) + self.launched_queries.append(query) + return query + + def wait_till_finish(self, handle): + while not self[handle].finished: + time.sleep(1) + return self[handle] + + def sanitize_response(self, resp): + resp.raise_for_status() + try: + resp_json = resp.json(object_hook=WrappedJson) + if 'lensAPIResult' in resp_json: + resp_json = resp_json.lens_a_p_i_result + if 'error' in resp_json: + raise Exception(resp_json['error']) + if 'data' in resp_json: + data = resp_json.data + if len(data) == 2 and 'type' in data: + keys = list(data.keys()) + keys.remove('type') + return WrappedJson({data['type']: data[keys[0]]}) + return data + except: + resp_json = resp.json() + return resp_json + + http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/lens/client/utils.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/lens/client/utils.py b/contrib/clients/python/lens/client/utils.py new file mode 100644 index 0000000..16993ca --- /dev/null +++ b/contrib/clients/python/lens/client/utils.py @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import xml.etree.ElementTree as ET + +def to_camel_case(snake_str): + """ + Converts snake_case to camelCase. + :param snake_str: a string in snake case + :return: equivalent string in camel case + + >>> to_camel_case('a_b') + 'aB' + >>> to_camel_case('some_important_string') + 'someImportantString' + """ + + components = snake_str.split('_') + # We capitalize the first letter of each component except the first one + # with the 'title' method and join them together. + return components[0] + "".join(x.title() for x in components[1:]) + + +def conf_to_xml(conf): + """ + Converts LensConf given as dictionary to xml string + :param conf: a Dictionary + :return: LensConf xml string representation + >>> conf_to_xml(None) + '<conf></conf>' + >>> conf_to_xml({}) + '<conf></conf>' + >>> conf_to_xml({'a':'b'}) + '<conf><properties><entry><key>a</key><value>b</value></entry></properties></conf>' + """ + if conf and len(conf) != 0: + return "<conf><properties>" + \ + "".join("<entry><key>%s</key><value>%s</value></entry>" % (k, v) for k, v in conf.items()) + \ + "</properties></conf>" + return "<conf></conf>" + +def xml_file_to_conf(path): + """ + Reads file present at path as hadoop configuration file and converts that to a dictionary. + :param path: Path of xml + :return: configuration as dictionary + + """ + return {property.find('name').text: property.find('value').text for property in ET.parse(path).getroot()} http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/clients/python/pom.xml b/contrib/clients/python/pom.xml new file mode 100644 index 0000000..79b198c --- /dev/null +++ b/contrib/clients/python/pom.xml @@ -0,0 +1,110 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <name>Lens Python Client</name> + <parent> + <artifactId>lens-contrib-clients</artifactId> + <groupId>org.apache.lens</groupId> + <version>2.6.0-beta-SNAPSHOT</version> + </parent> + + <artifactId>lens-python-client</artifactId> + <packaging>pom</packaging> + <description>Lens python clients jar</description> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <configuration> + <executable>python</executable> + <workingDirectory>${basedir}</workingDirectory> + <environmentVariables> + <VERSION>${project.version}</VERSION> + <!--<BUILD_NUMBER>${buildNumber}</BUILD_NUMBER>--> + <!--<TIMESTAMP>${timestamp}</TIMESTAMP>--> + </environmentVariables> + </configuration> + <executions> + <execution> + <id>setuptools clean</id> + <phase>clean</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <arguments> + <argument>setup.py</argument> + <argument>clean</argument> + <argument>--all</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>setuptools install</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <arguments> + <argument>setup.py</argument> + <argument>develop</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>setuptools test</id> + <phase>test</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <skip>${maven.test.skip}</skip> + <arguments> + <argument>setup.py</argument> + <argument>test</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>setuptools deploy</id> + <phase>deploy</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <arguments> + <argument>setup.py</argument> + <argument>register</argument> + <argument>sdist</argument> + <argument>upload</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/setup.cfg ---------------------------------------------------------------------- diff --git a/contrib/clients/python/setup.cfg b/contrib/clients/python/setup.cfg new file mode 100644 index 0000000..5ad322b --- /dev/null +++ b/contrib/clients/python/setup.cfg @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[metadata] +description-file = README http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/setup.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/setup.py b/contrib/clients/python/setup.py new file mode 100644 index 0000000..6147c90 --- /dev/null +++ b/contrib/clients/python/setup.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function + +import io +import os +import sys + +from setuptools import setup +from setuptools.command.test import test as TestCommand + +here = os.path.abspath(os.path.dirname(__file__)) + +def read(*filenames, **kwargs): + encoding = kwargs.get('encoding', 'utf-8') + sep = kwargs.get('sep', '\n') + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + +long_description = read('README') + +class Tox(TestCommand): + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + def run_tests(self): + #import here, cause outside the eggs aren't loaded + import tox + errcode = tox.cmdline(self.test_args) + sys.exit(errcode) + +setup( + name='lens-client', + version="2.6.0", + url='http://github.com/apache/lens/', + license='Apache Software License', + author='Apache', + tests_require=['tox'], + install_requires=['requests>=2.9.1', 'six>=1.10.0'], + cmdclass={'test': Tox}, + author_email='[email protected]', + description='Python Lens Client', + long_description=long_description, + packages=['lens', 'lens.client'], + include_package_data=True, + platforms='any', + test_suite='lens.client.test.test_lensclient', + classifiers = [ + 'Programming Language :: Python', + 'Development Status :: 4 - Beta', + 'Natural Language :: English', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + ], + extras_require={ + 'testing': ['pytest'], + } +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/test/test_lensclient.py ---------------------------------------------------------------------- diff --git a/contrib/clients/python/test/test_lensclient.py b/contrib/clients/python/test/test_lensclient.py new file mode 100644 index 0000000..d49c55b --- /dev/null +++ b/contrib/clients/python/test/test_lensclient.py @@ -0,0 +1,161 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function + +import random +import string +from contextlib import contextmanager +from requests.exceptions import HTTPError + +import time + +import pytest +from lens.client import LensClient +from lens.client.models import WrappedJson +import subprocess +import os +import glob + +def check_output(command): + output = subprocess.check_output(command.split()) + if isinstance(output, bytes): # For Python 3. Python 2 directly gives string + output = output.decode("utf-8") + return output + +@contextmanager +def cwd(dir): + cur_dir = os.getcwd() + os.chdir(dir) + yield + os.chdir(cur_dir) + +def time_sorted_ls(path): + mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime + return list(sorted(os.listdir(path), key=mtime)) + +def has_error(msg): + return any(x in msg for x in ('Error', 'error', 'Exception', 'exception')) + +def get_error(): + latest_out_file = list(name for name in time_sorted_ls('logs') if 'lensserver.out' in name)[-1] + print (latest_out_file) + with open(os.path.join('logs', latest_out_file)) as f: + return f.read() + +def select_query(path): + with open(path) as f: + for line in f: + if 'cube select' in line and 'sample_cube' in line: + return line + +class TestLensClient(object): + @classmethod + def setup_class(cls): + cls.db = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + try: + root = check_output("git rev-parse --show-toplevel") + except: + root = os.getenv("LENS_GIT_PATH") + print(root, root.strip()) + joined = os.path.join(root.strip(), 'lens-dist', 'target', '*bin', '*bin') + cls.base_path = glob.glob(joined)[0] + with cwd(cls.base_path): + with cwd('server'): + server_start_output = check_output("bin/lens-ctl restart") + assert "Started lens server" in server_start_output + assert os.path.exists('logs/server.pid') + time.sleep(1) + while not os.path.exists('logs/lensserver.log'): + error = get_error() + if has_error(error): + # Assert again with complete error + assert os.path.exists('logs/lensserver.log'), error + time.sleep(1) + error = get_error() + if has_error(error): + assert False, error + + with cwd('client'): + cls.candidate_query = select_query('examples/resources/cube-queries.sql') + with open('check_connection.sql', 'w') as f: + f.write('show databases') + for i in range(100): + try: + output = check_output('bin/lens-cli.sh --cmdfile check_connection.sql') + if not has_error(output): + break + except: + # Ignore error and retry + pass + time.sleep(1) + os.remove('check_connection.sql') + create_output = check_output('bin/run-examples.sh sample-metastore -db ' + cls.db) + if has_error(create_output): + raise Exception("Couldn't create sample metastore: " + create_output) + populate_output = check_output('bin/run-examples.sh populate-metastore -db ' + cls.db) + if has_error(populate_output): + raise Exception("Couldn't populate sample metastore: " + populate_output) + @classmethod + def teardown_class(cls): + # TODO: drop database + with cwd(cls.base_path): + with cwd('client'): + with open('drop_db.sql', 'w') as f: + f.write("drop database {db} --cascade".format(db=cls.db)) + drop_output = check_output('bin/lens-cli.sh --cmdfile drop_db.sql') + if has_error(drop_output): + raise Exception("Couldn't drop db") + os.remove('drop_db.sql') + with cwd('server'): + stop_output = check_output('bin/lens-ctl stop') + if has_error(stop_output): + raise("Error stopping server: " + stop_output) + + def get_client(self): + return LensClient(database = self.db, conf=os.path.join(self.base_path, 'client', 'conf')) + + def test_auto_close_session(self): + with self.get_client() as client: + pass + with pytest.raises(HTTPError) as e: + # Now any api should give 410 + client.queries(state='RUNNING') + assert e.value.response.status_code == 410 + + def test_wrong_query(self): + with self.get_client() as client: + with pytest.raises(HTTPError) as e: + client.queries.submit("blah") + assert e.value.response.status_code == 400; + assert 'Syntax Error' in e.value.response.json(object_hook=WrappedJson).lens_a_p_i_result.error.message + + def test_submit_query(self): + with self.get_client() as client: + handle = client.queries.submit(self.candidate_query) + # session not closed + assert client.queries[handle] + client.queries.wait_till_finish(handle) + client.close_session() + + def test_list_query(self): + with self.get_client() as client: + handle = client.queries.submit(self.candidate_query, query_name="Candidate Query") + finished_query = client.queries.wait_till_finish(handle) + assert client.queries[handle] == finished_query + queries = client.queries(state='SUCCESSFUL', fromDate=finished_query.submission_time - 1, toDate=finished_query.submission_time + 1) + assert handle in queries + http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/clients/python/tox.ini ---------------------------------------------------------------------- diff --git a/contrib/clients/python/tox.ini b/contrib/clients/python/tox.ini new file mode 100644 index 0000000..c048e14 --- /dev/null +++ b/contrib/clients/python/tox.ini @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[tox] +envlist=py27, py35 + +[testenv] +deps= + pytest + coverage + pytest-cov +setenv= + PYTHONWARNINGS=all +passenv = + HADOOP_HOME + HIVE_HOME + LENS_GIT_PATH + +[pytest] +addopts=--ignore setup.py --doctest-modules +norecursedirs=.tox .git + +[testenv:py27] +commands= + py.test + +[testenv:py35] +commands= + py.test + +[testenv:py27verbose] +basepython=python +commands= + py.test --cov=. --cov-report term + +[testenv:py35verbose] +basepython=python3.5 +commands= + py.test --cov=. --cov-report term \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/contrib/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/pom.xml b/contrib/pom.xml new file mode 100644 index 0000000..0f792ab --- /dev/null +++ b/contrib/pom.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <name>Lens Contrib</name> + <parent> + <artifactId>apache-lens</artifactId> + <groupId>org.apache.lens</groupId> + <version>2.6.0-beta-SNAPSHOT</version> + </parent> + + <artifactId>lens-contrib</artifactId> + <packaging>pom</packaging> + <description>Lens Contrib jar</description> + <modules> + <module>clients</module> + </modules> +</project> http://git-wip-us.apache.org/repos/asf/lens/blob/12b95da0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9695126..f1b2289 100644 --- a/pom.xml +++ b/pom.xml @@ -657,6 +657,11 @@ <exclude>**/*.js</exclude> <exclude>**/*.properties</exclude> <exclude>**/*.json</exclude> + <!-- python client --> + <exclude>**/.tox/</exclude> + <exclude>**/.cache/</exclude> + <exclude>**/*.egg-info/**</exclude> + <exclude>**/*.egg</exclude> </excludes> </configuration> <executions>
