Repository: bigtop Updated Branches: refs/heads/master 1357959c0 -> 59fa03c0f
BIGTOP-1636. Missing patches and files for hue Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e392f6a1 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e392f6a1 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e392f6a1 Branch: refs/heads/master Commit: e392f6a1652652dfe1d4149f145b8b94e1d6fb85 Parents: 1357959 Author: Olaf Flebbe <[email protected]> Authored: Thu Jan 29 21:15:53 2015 +0100 Committer: Olaf Flebbe <[email protected]> Committed: Thu Jan 29 21:50:27 2015 +0100 ---------------------------------------------------------------------- .../src/common/hue/patch0-getlog.diff | 264 +++++++++++++++++++ .../src/common/hue/patch1-bigtop-conf.diff | 24 ++ bigtop-packages/src/deb/hue/hue-impala.install | 1 + .../src/deb/hue/hue-security.install | 1 + 4 files changed, 290 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e392f6a1/bigtop-packages/src/common/hue/patch0-getlog.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hue/patch0-getlog.diff b/bigtop-packages/src/common/hue/patch0-getlog.diff new file mode 100644 index 0000000..957c6ab --- /dev/null +++ b/bigtop-packages/src/common/hue/patch0-getlog.diff @@ -0,0 +1,264 @@ +commit 6a0246710f7deeb0fd2e1f2b3b209ad119c30b72 +Author: Peter Slawski <[email protected]> +Date: Fri Oct 10 21:11:28 2014 -0700 + + HUE-2387 [beeswax] Support getting logs using FetchResults() api + +diff --git a/apps/beeswax/gen-py/TCLIService/ttypes.py b/apps/beeswax/gen-py/TCLIService/ttypes.py +index b273bf0..d1e1a5e 100644 +--- a/apps/beeswax/gen-py/TCLIService/ttypes.py ++++ b/apps/beeswax/gen-py/TCLIService/ttypes.py +@@ -5124,6 +5124,7 @@ class TFetchResultsReq(object): + - operationHandle + - orientation + - maxRows ++ - fetchType + """ + + thrift_spec = ( +@@ -5131,12 +5132,14 @@ class TFetchResultsReq(object): + (1, TType.STRUCT, 'operationHandle', (TOperationHandle, TOperationHandle.thrift_spec), None, ), # 1 + (2, TType.I32, 'orientation', None, 0, ), # 2 + (3, TType.I64, 'maxRows', None, None, ), # 3 ++ (4, TType.I16, 'fetchType', None, 0, ), # 4 + ) + +- def __init__(self, operationHandle=None, orientation=thrift_spec[2][4], maxRows=None,): ++ def __init__(self, operationHandle=None, orientation=thrift_spec[2][4], maxRows=None, fetchType=thrift_spec[4][4],): + self.operationHandle = operationHandle + self.orientation = orientation + self.maxRows = maxRows ++ self.fetchType = fetchType + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: +@@ -5163,6 +5166,11 @@ class TFetchResultsReq(object): + self.maxRows = iprot.readI64(); + else: + iprot.skip(ftype) ++ elif fid == 4: ++ if ftype == TType.I16: ++ self.fetchType = iprot.readI16(); ++ else: ++ iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() +@@ -5185,6 +5193,10 @@ class TFetchResultsReq(object): + oprot.writeFieldBegin('maxRows', TType.I64, 3) + oprot.writeI64(self.maxRows) + oprot.writeFieldEnd() ++ if self.fetchType is not None: ++ oprot.writeFieldBegin('fetchType', TType.I16, 4) ++ oprot.writeI16(self.fetchType) ++ oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + +diff --git a/apps/beeswax/src/beeswax/api.py b/apps/beeswax/src/beeswax/api.py +index 91da7dd..f56b44a 100644 +--- a/apps/beeswax/src/beeswax/api.py ++++ b/apps/beeswax/src/beeswax/api.py +@@ -172,7 +172,8 @@ def watch_query_refresh_json(request, id): + handle, state = _get_query_handle_and_state(query_history) + + try: +- log = db.get_log(handle) ++ start_over = request.POST.get('log-start-over') == 'true' ++ log = db.get_log(handle, start_over=start_over) + except Exception, ex: + log = str(ex) + +diff --git a/apps/beeswax/src/beeswax/conf.py b/apps/beeswax/src/beeswax/conf.py +index 1667481..e7ffc8e 100644 +--- a/apps/beeswax/src/beeswax/conf.py ++++ b/apps/beeswax/src/beeswax/conf.py +@@ -60,6 +60,14 @@ SERVER_CONN_TIMEOUT = Config( + type=int, + help=_t('Timeout in seconds for Thrift calls.')) + ++USE_GET_LOG_API = Config( ++ key='use_get_log_api', ++ default=True, ++ type=coerce_bool, ++ help=_t('Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.' ++ 'If false, Hue will use the FetchResults() thrift call instead.') ++) ++ + BROWSE_PARTITIONED_TABLE_LIMIT = Config( + key='browse_partitioned_table_limit', + default=250, +diff --git a/apps/beeswax/src/beeswax/server/dbms.py b/apps/beeswax/src/beeswax/server/dbms.py +index baaba68..86b012b 100644 +--- a/apps/beeswax/src/beeswax/server/dbms.py ++++ b/apps/beeswax/src/beeswax/server/dbms.py +@@ -338,8 +338,8 @@ class HiveServer2Dbms(object): + return self.client.use(query) + + +- def get_log(self, query_handle): +- return self.client.get_log(query_handle) ++ def get_log(self, query_handle, start_over=True): ++ return self.client.get_log(query_handle, start_over) + + + def get_state(self, handle): +diff --git a/apps/beeswax/src/beeswax/server/hive_server2_lib.py b/apps/beeswax/src/beeswax/server/hive_server2_lib.py +index d7c222c..f7f2562 100644 +--- a/apps/beeswax/src/beeswax/server/hive_server2_lib.py ++++ b/apps/beeswax/src/beeswax/server/hive_server2_lib.py +@@ -14,6 +14,7 @@ + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. ++from itertools import imap + + import logging + import re +@@ -590,6 +591,15 @@ class HiveServerClient: + return res, schema + + ++ def fetch_log(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=1000): ++ req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation, ++ maxRows=max_rows, fetchType=1) ++ res = self.call(self._client.FetchResults, req) ++ ++ lines = imap(lambda r: r.colVals[0].stringVal.value, res.results.rows) ++ return '\n'.join(lines) ++ ++ + def get_operation_status(self, operation_handle): + req = TGetOperationStatusReq(operationHandle=operation_handle) + return self.call(self._client.GetOperationStatus, req) +@@ -759,9 +769,18 @@ class HiveServerClientCompatible(object): + return 'Does not exist in HS2' + + +- def get_log(self, handle): ++ def get_log(self, handle, start_over=True): + operationHandle = handle.get_rpc_handle() +- return self._client.get_log(operationHandle) ++ ++ if beeswax_conf.USE_GET_LOG_API.get() or self.query_server['server_name'] == 'impala': ++ return self._client.get_log(operationHandle) ++ else: ++ if start_over: ++ orientation = TFetchOrientation.FETCH_FIRST ++ else: ++ orientation = TFetchOrientation.FETCH_NEXT ++ ++ return self._client.fetch_log(operationHandle, orientation=orientation, max_rows=-1) + + + def get_databases(self): +diff --git a/apps/beeswax/src/beeswax/templates/execute.mako b/apps/beeswax/src/beeswax/templates/execute.mako +index 6b1f086..27909b0 100644 +--- a/apps/beeswax/src/beeswax/templates/execute.mako ++++ b/apps/beeswax/src/beeswax/templates/execute.mako +@@ -2637,6 +2637,11 @@ function setupCodeMirrorSubscription() { + + // Knockout + viewModel = new BeeswaxViewModel("${app_name}"); ++ ++% if not beeswax_conf.USE_GET_LOG_API.get() and app_name != 'impala': ++ viewModel.shouldAppendLogs = true; ++% endif ++ + % if query_history: + loadQueryHistory(${query_history.id}); + % elif design.id: +diff --git a/apps/beeswax/static/js/beeswax.vm.js b/apps/beeswax/static/js/beeswax.vm.js +index fdd2f78..6e6c65c 100644 +--- a/apps/beeswax/static/js/beeswax.vm.js ++++ b/apps/beeswax/static/js/beeswax.vm.js +@@ -560,7 +560,8 @@ function BeeswaxViewModel(server) { + self.watchQuery = function() { + var data = { + 'query-query': self.design.query.value(), +- 'query-database': self.database() ++ 'query-database': self.database(), ++ 'log-start-over': self.design.watch.logs().length == 0 + }; + $.extend(data, self.getSettingsFormData()); + $.extend(data, self.getFileResourcesFormData()); +@@ -589,6 +590,17 @@ function BeeswaxViewModel(server) { + $.ajax(request); + }; + ++ self.shouldAppendLogs = false; ++ ++ self.applyLogs = function(log) { ++ var lines = log.split("\n") ++ ++ if (self.shouldAppendLogs) { ++ lines = self.design.watch.logs().concat(lines); ++ } ++ self.design.watch.logs(lines); ++ }; ++ + self.watchQueryLoop = function(fn) { + var TIMEOUT = 100; + var timer = null; +@@ -607,7 +619,7 @@ function BeeswaxViewModel(server) { + clearTimeout(timer); + self.design.isRunning(false); + if (data.log) { +- self.design.watch.logs(data.log.split("\n")); ++ self.applyLogs(data.log) + // scroll logs + self.design.watch.jobUrls(data.jobUrls); + } +@@ -623,7 +635,7 @@ function BeeswaxViewModel(server) { + } else { + self.design.statement(data.statement); // In case new no result statement executed + if (data.log) { +- self.design.watch.logs(data.log.split("\n")); ++ self.applyLogs(data.log) + // scroll logs + self.design.watch.jobUrls(data.jobUrls); + } +diff --git a/apps/beeswax/thrift/TCLIService.thrift b/apps/beeswax/thrift/TCLIService.thrift +index 53ea3cf..63dad7e 100644 +--- a/apps/beeswax/thrift/TCLIService.thrift ++++ b/apps/beeswax/thrift/TCLIService.thrift +@@ -1002,6 +1002,9 @@ struct TFetchResultsReq { + // Max number of rows that should be returned in + // the rowset. + 3: required i64 maxRows ++ ++ // The type of a fetch results request. 0 represents Query output. 1 represents Log ++ 4: optional i16 fetchType = 0 + } + + struct TFetchResultsResp { +diff --git a/desktop/conf.dist/hue.ini b/desktop/conf.dist/hue.ini +index dc06a85..98c6bbb 100644 +--- a/desktop/conf.dist/hue.ini ++++ b/desktop/conf.dist/hue.ini +@@ -724,6 +724,10 @@ + # Timeout in seconds for thrift calls to Hive service + ## server_conn_timeout=120 + ++ # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. ++ # If false, Hue will use the FetchResults() thrift call instead. ++ ## use_get_log_api=true ++ + # Set a LIMIT clause when browsing a partitioned table. + # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. + ## browse_partitioned_table_limit=250 +diff --git a/desktop/conf/pseudo-distributed.ini.tmpl b/desktop/conf/pseudo-distributed.ini.tmpl +index 784c235..52f719b 100644 +--- a/desktop/conf/pseudo-distributed.ini.tmpl ++++ b/desktop/conf/pseudo-distributed.ini.tmpl +@@ -731,6 +731,10 @@ + # Timeout in seconds for thrift calls to Hive service + ## server_conn_timeout=120 + ++ # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. ++ # If false, Hue will use the FetchResults() thrift call instead. ++ ## use_get_log_api=true ++ + # Set a LIMIT clause when browsing a partitioned table. + # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. + ## browse_partitioned_table_limit=250 http://git-wip-us.apache.org/repos/asf/bigtop/blob/e392f6a1/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff b/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff new file mode 100644 index 0000000..36e1532 --- /dev/null +++ b/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff @@ -0,0 +1,24 @@ +diff --git a/desktop/conf.dist/hue.ini~ b/desktop/conf.dist/hue.ini +index 9b30901..a2b9bea 100644 +--- a/desktop/conf.dist/hue.ini~ ++++ b/desktop/conf.dist/hue.ini +@@ -95,7 +95,8 @@ + + # Comma separated list of apps to not load at server startup. + # e.g.: pig,zookeeper +- ## app_blacklist= ++ # Bigtop does not bundle impala ++ app_blacklist=impala + + # The directory where to store the auditing logs. Auditing is disable if the value is empty. + # e.g. /var/log/hue/audit.log +@@ -722,7 +723,8 @@ + + # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. + # If false, Hue will use the FetchResults() thrift call instead. +- ## use_get_log_api=true ++ # Bigtop Hive only supports FetchResults() ++ use_get_log_api=false + + # Set a LIMIT clause when browsing a partitioned table. + # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. http://git-wip-us.apache.org/repos/asf/bigtop/blob/e392f6a1/bigtop-packages/src/deb/hue/hue-impala.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hue/hue-impala.install b/bigtop-packages/src/deb/hue/hue-impala.install new file mode 100644 index 0000000..1ff1a4e --- /dev/null +++ b/bigtop-packages/src/deb/hue/hue-impala.install @@ -0,0 +1 @@ +/usr/lib/hue/apps/impala http://git-wip-us.apache.org/repos/asf/bigtop/blob/e392f6a1/bigtop-packages/src/deb/hue/hue-security.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hue/hue-security.install b/bigtop-packages/src/deb/hue/hue-security.install new file mode 100644 index 0000000..86816f6 --- /dev/null +++ b/bigtop-packages/src/deb/hue/hue-security.install @@ -0,0 +1 @@ +/usr/lib/hue/apps/security
