[SPOT-213][SPOT-250][OA][DATA] temp fix for impala calls, add TODO for impyla conversion
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0e749191 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0e749191 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0e749191 Branch: refs/heads/master Commit: 0e749191311a3c1695cd40322c1b5788cc56e50c Parents: d1f5a67 Author: natedogs911 <natedogs...@gmail.com> Authored: Thu Jan 18 11:06:50 2018 -0800 Committer: natedogs911 <natedogs...@gmail.com> Committed: Thu Jan 18 11:06:50 2018 -0800 ---------------------------------------------------------------------- spot-oa/oa/components/data/hive.py | 1 + spot-oa/oa/components/data/impala.py | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0e749191/spot-oa/oa/components/data/hive.py ---------------------------------------------------------------------- diff --git a/spot-oa/oa/components/data/hive.py b/spot-oa/oa/components/data/hive.py index a7c1d4b..7d2eaa2 100644 --- a/spot-oa/oa/components/data/hive.py +++ b/spot-oa/oa/components/data/hive.py @@ -24,6 +24,7 @@ class Engine(object): self._pipeline = pipeline def query(self,query,output_file=None, delimiter=','): + # TODO: fix kerberos compatibility, use impyla hive_config = "set mapred.max.split.size=1073741824;set hive.exec.reducers.max=10;set hive.cli.print.header=true;" del_format = "| sed 's/[\t]/{0}/g'".format(delimiter) http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0e749191/spot-oa/oa/components/data/impala.py ---------------------------------------------------------------------- diff --git a/spot-oa/oa/components/data/impala.py b/spot-oa/oa/components/data/impala.py index bfc1c5a..10d1f5b 100644 --- a/spot-oa/oa/components/data/impala.py +++ b/spot-oa/oa/components/data/impala.py @@ -16,6 +16,8 @@ # from subprocess import check_output +from common import configurator + class Engine(object): @@ -24,17 +26,32 @@ class Engine(object): self._daemon_node = conf['impala_daemon'] self._db = db self._pipeline = pipeline - impala_cmd = "impala-shell -i {0} --quiet -q 'INVALIDATE METADATA {1}.{2}'".format(self._daemon_node,self._db, self._pipeline) + + if configurator.kerberos_enabled(): + self._impala_shell = "impala-shell -k -i {0} --quiet".format(self._daemon_node) + else: + self._impala_shell = "impala-shell -i {0} --quiet".format(self._daemon_node) + + impala_cmd = "{0} -q 'INVALIDATE METADATA {1}.{2}'".format(self._impala_shell, self._db, self._pipeline) check_output(impala_cmd,shell=True) - impala_cmd = "impala-shell -i {0} --quiet -q 'REFRESH {1}.{2}'".format(self._daemon_node,self._db, self._pipeline) + impala_cmd = "{0} -q 'REFRESH {1}.{2}'".format(self._impala_shell, self._db, self._pipeline) check_output(impala_cmd,shell=True) def query(self,query,output_file=None,delimiter=","): if output_file: - impala_cmd = "impala-shell -i {0} --quiet --print_header -B --output_delimiter='{1}' -q \"{2}\" -o {3}".format(self._daemon_node,delimiter,query,output_file) + impala_cmd = "{0} --print_header -B --output_delimiter='{1}' -q \"{2}\" -o {3}".format( + self._impala_shell, + delimiter, + query, + output_file + ) else: - impala_cmd = "impala-shell -i {0} --quiet --print_header -B --output_delimiter='{1}' -q \"{2}\"".format(self._daemon_node,delimiter,query) + impala_cmd = "{0} --print_header -B --output_delimiter='{1}' -q \"{2}\"".format( + self._impala_shell, + delimiter, + query + ) check_output(impala_cmd,shell=True)
