METRON-1389 Zeppelin notebook import does not work with Ambari 2.6 (anandsubbu) closes apache/metron#888
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c9437c4 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c9437c4 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c9437c4 Branch: refs/heads/feature/METRON-1344-test-infrastructure Commit: 1c9437c41c4eb626bd44193ec6545f9356f3f7b2 Parents: a285b83 Author: anandsubbu <[email protected]> Authored: Tue Jan 30 20:39:45 2018 +0530 Committer: anandsubbu <[email protected]> Committed: Tue Jan 30 20:39:45 2018 +0530 ---------------------------------------------------------------------- .../package/scripts/indexing_commands.py | 48 ++++++++++++++++++++ .../CURRENT/package/scripts/indexing_master.py | 14 ++++-- .../package/scripts/params/params_linux.py | 1 + .../package/scripts/params/status_params.py | 1 + 4 files changed, 61 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index c057b72..4c862f0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -16,12 +16,15 @@ limitations under the License. """ import os +import re +import requests import time from datetime import datetime from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File +from resource_management.libraries.functions import format as ambari_format import metron_service import metron_security @@ -327,3 +330,48 @@ class IndexingCommands: raise Fail("Indexing topology not running") Logger.info("Indexing service check completed successfully") + + def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env): + """ + With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password + for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). + + This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the + session connection details to be used for importing Zeppelin notebooks. + :param ses: Session handle + :param zeppelin_server_url: Zeppelin Server URL + :return: ses + """ + from params import params + env.set_params(params) + + # Check if authentication is enabled on the Zeppelin server + try: + ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) + + # Establish connection if authentication is enabled + try: + Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") + # Read the Shiro admin user credentials from Zeppelin config in Ambari + seen_users = False + username = None + password = None + if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): + seen_users = True + tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() + userpassword = tokens.split(',')[0].strip() + username = userpassword.split('=')[0].strip() + password = userpassword.split('=')[1].strip() + else: + Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") + + zeppelin_payload = {'userName': username, 'password' : password} + ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) + except: + pass + + # If authentication is not enabled, fall back to default method of imporing notebooks + except requests.exceptions.RequestException: + ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) + + return ses http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 18d5224..ce8c074 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -15,6 +15,7 @@ limitations under the License. """ import os +import requests from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute @@ -151,14 +152,21 @@ class Indexing(Script): def zeppelin_notebook_import(self, env): from params import params env.set_params(params) + commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) + + # Check if authentication is configured on Zeppelin server, and fetch details if enabled. + ses = requests.session() + ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): - zeppelin_cmd = ambari_format( - 'curl -s -XPOST http://{zeppelin_server_url}/api/notebook/import -d "@' + os.path.join(dirName, fileName) + '"') - Execute(zeppelin_cmd, logoutput=True) + Logger.info("Importing notebook: " + fileName) + zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import') + zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')} + res = ses.post(zeppelin_import_url, files=zeppelin_notebook) + Logger.info("Result: " + res.text) if __name__ == "__main__": Indexing().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 0d5b721..3506fab 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -202,6 +202,7 @@ meta_index_path = tmp_dir + "/metaalert_index.template" # Zeppelin Notebooks metron_config_zeppelin_path = format("{metron_config_path}/zeppelin") +zeppelin_shiro_ini_content = status_params.zeppelin_shiro_ini_content # kafka_security kafka_security_protocol = config['configurations']['kafka-broker'].get('security.inter.broker.protocol', 'PLAINTEXT') http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index b43c30c..62cfc7a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -95,6 +95,7 @@ storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] # Zeppelin zeppelin_server_url = config['configurations']['metron-env']['zeppelin_server_url'] +zeppelin_shiro_ini_content = config['configurations']['zeppelin-shiro-ini']['shiro_ini_content'] # Security stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
