METRON-1389 Zeppelin notebook import does not work with Ambari 2.6 (anandsubbu) 
closes apache/metron#888


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c9437c4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c9437c4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c9437c4

Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 1c9437c41c4eb626bd44193ec6545f9356f3f7b2
Parents: a285b83
Author: anandsubbu <anandsu...@gmail.com>
Authored: Tue Jan 30 20:39:45 2018 +0530
Committer: anandsubbu <anandsu...@apache.org>
Committed: Tue Jan 30 20:39:45 2018 +0530

----------------------------------------------------------------------
 .../package/scripts/indexing_commands.py        | 48 ++++++++++++++++++++
 .../CURRENT/package/scripts/indexing_master.py  | 14 ++++--
 .../package/scripts/params/params_linux.py      |  1 +
 .../package/scripts/params/status_params.py     |  1 +
 4 files changed, 61 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index c057b72..4c862f0 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -16,12 +16,15 @@ limitations under the License.
 """
 
 import os
+import re
+import requests
 import time
 
 from datetime import datetime
 from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute, File
+from resource_management.libraries.functions import format as ambari_format
 
 import metron_service
 import metron_security
@@ -327,3 +330,48 @@ class IndexingCommands:
             raise Fail("Indexing topology not running")
 
         Logger.info("Indexing service check completed successfully")
+
+    def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env):
+        """
+        With Ambari 2.5+, Zeppelin server is enabled to work with Shiro 
authentication, which requires user/password
+        for authentication (see 
https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for 
details).
+
+        This method checks if Shiro authentication is enabled on the Zeppelin 
server. And if enabled, it returns the
+        session connection details to be used for importing Zeppelin notebooks.
+        :param ses: Session handle
+        :param zeppelin_server_url: Zeppelin Server URL
+        :return: ses
+        """
+        from params import params
+        env.set_params(params)
+
+        # Check if authentication is enabled on the Zeppelin server
+        try:
+            ses.get(ambari_format('http://{zeppelin_server_url}/api/login'))
+
+            # Establish connection if authentication is enabled
+            try:
+                Logger.info("Shiro authentication is found to be enabled on 
the Zeppelin server.")
+                # Read the Shiro admin user credentials from Zeppelin config 
in Ambari
+                seen_users = False
+                username = None
+                password = None
+                if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, 
re.MULTILINE):
+                    seen_users = True
+                    tokens = re.search(r'^admin\ =.*', 
params.zeppelin_shiro_ini_content, re.MULTILINE).group()
+                    userpassword = tokens.split(',')[0].strip()
+                    username = userpassword.split('=')[0].strip()
+                    password = userpassword.split('=')[1].strip()
+                else:
+                    Logger.error("ERROR: Admin credentials config was not 
found in shiro.ini. Notebook import may fail.")
+
+                zeppelin_payload = {'userName': username, 'password' : 
password}
+                
ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), 
data=zeppelin_payload)
+            except:
+                pass
+
+        # If authentication is not enabled, fall back to default method of 
imporing notebooks
+        except requests.exceptions.RequestException:
+            ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook'))
+
+        return ses

http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 18d5224..ce8c074 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -15,6 +15,7 @@ limitations under the License.
 """
 
 import os
+import requests
 from resource_management.core.exceptions import ComponentIsNotRunning
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
@@ -151,14 +152,21 @@ class Indexing(Script):
     def zeppelin_notebook_import(self, env):
         from params import params
         env.set_params(params)
+        commands = IndexingCommands(params)
 
         Logger.info(ambari_format('Searching for Zeppelin Notebooks in 
{metron_config_zeppelin_path}'))
+
+        # Check if authentication is configured on Zeppelin server, and fetch 
details if enabled.
+        ses = requests.session()
+        ses = commands.get_zeppelin_auth_details(ses, 
params.zeppelin_server_url, env)
         for dirName, subdirList, files in 
os.walk(params.metron_config_zeppelin_path):
             for fileName in files:
                 if fileName.endswith(".json"):
-                    zeppelin_cmd = ambari_format(
-                        'curl -s -XPOST 
http://{zeppelin_server_url}/api/notebook/import -d "@' + os.path.join(dirName, 
fileName) + '"')
-                    Execute(zeppelin_cmd, logoutput=True)
+                    Logger.info("Importing notebook: " + fileName)
+                    zeppelin_import_url = 
ambari_format('http://{zeppelin_server_url}/api/notebook/import')
+                    zeppelin_notebook = {'file' : open(os.path.join(dirName, 
fileName), 'rb')}
+                    res = ses.post(zeppelin_import_url, 
files=zeppelin_notebook)
+                    Logger.info("Result: " + res.text)
 
 if __name__ == "__main__":
     Indexing().execute()

http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 0d5b721..3506fab 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -202,6 +202,7 @@ meta_index_path = tmp_dir + "/metaalert_index.template"
 
 # Zeppelin Notebooks
 metron_config_zeppelin_path = format("{metron_config_path}/zeppelin")
+zeppelin_shiro_ini_content = status_params.zeppelin_shiro_ini_content
 
 # kafka_security
 kafka_security_protocol = 
config['configurations']['kafka-broker'].get('security.inter.broker.protocol', 
'PLAINTEXT')

http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index b43c30c..62cfc7a 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -95,6 +95,7 @@ storm_rest_addr = 
config['configurations']['metron-env']['storm_rest_addr']
 
 # Zeppelin
 zeppelin_server_url = 
config['configurations']['metron-env']['zeppelin_server_url']
+zeppelin_shiro_ini_content = 
config['configurations']['zeppelin-shiro-ini']['shiro_ini_content']
 
 # Security
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])

Reply via email to