Repository: bigtop Updated Branches: refs/heads/master 0c0e2fca4 -> f83a9c48a
BIGTOP-1850: Update Hue to 3.8 Signed-off-by: Evans Ye <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/d4965671 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/d4965671 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/d4965671 Branch: refs/heads/master Commit: d4965671700df504cf22cf5ccd029eb16f012947 Parents: 0c0e2fc Author: Olaf Flebbe <[email protected]> Authored: Mon May 4 22:00:51 2015 +0200 Committer: Evans Ye <[email protected]> Committed: Sat May 9 15:40:58 2015 +0000 ---------------------------------------------------------------------- .../src/common/hue/hue-virtualenv.tgz | Bin 1769776 -> 0 bytes .../src/common/hue/patch0-getlog.diff | 264 ------------------- .../src/common/hue/patch1-bigtop-conf.diff | 24 -- .../hue/patch2-disable-google-analytics.diff | 13 - bigtop-packages/src/deb/hue/rules | 6 - bigtop-packages/src/rpm/hue/SPECS/hue.spec | 7 - bigtop.mk | 4 +- 7 files changed, 2 insertions(+), 316 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop-packages/src/common/hue/hue-virtualenv.tgz ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hue/hue-virtualenv.tgz b/bigtop-packages/src/common/hue/hue-virtualenv.tgz deleted file mode 100644 index 29b2a38..0000000 Binary files a/bigtop-packages/src/common/hue/hue-virtualenv.tgz and /dev/null differ http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop-packages/src/common/hue/patch0-getlog.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hue/patch0-getlog.diff b/bigtop-packages/src/common/hue/patch0-getlog.diff deleted file mode 100644 index 957c6ab..0000000 --- a/bigtop-packages/src/common/hue/patch0-getlog.diff +++ /dev/null @@ -1,264 +0,0 @@ -commit 6a0246710f7deeb0fd2e1f2b3b209ad119c30b72 -Author: Peter Slawski <[email protected]> -Date: Fri Oct 10 21:11:28 2014 -0700 - - HUE-2387 [beeswax] Support getting logs using FetchResults() api - -diff --git a/apps/beeswax/gen-py/TCLIService/ttypes.py b/apps/beeswax/gen-py/TCLIService/ttypes.py -index b273bf0..d1e1a5e 100644 ---- a/apps/beeswax/gen-py/TCLIService/ttypes.py -+++ b/apps/beeswax/gen-py/TCLIService/ttypes.py -@@ -5124,6 +5124,7 @@ class TFetchResultsReq(object): - - operationHandle - - orientation - - maxRows -+ - fetchType - """ - - thrift_spec = ( -@@ -5131,12 +5132,14 @@ class TFetchResultsReq(object): - (1, TType.STRUCT, 'operationHandle', (TOperationHandle, TOperationHandle.thrift_spec), None, ), # 1 - (2, TType.I32, 'orientation', None, 0, ), # 2 - (3, TType.I64, 'maxRows', None, None, ), # 3 -+ (4, TType.I16, 'fetchType', None, 0, ), # 4 - ) - -- def __init__(self, operationHandle=None, orientation=thrift_spec[2][4], maxRows=None,): -+ def __init__(self, operationHandle=None, orientation=thrift_spec[2][4], maxRows=None, fetchType=thrift_spec[4][4],): - self.operationHandle = operationHandle - self.orientation = orientation - self.maxRows = maxRows -+ self.fetchType = fetchType - - def read(self, iprot): - if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: -@@ -5163,6 +5166,11 @@ class TFetchResultsReq(object): - self.maxRows = iprot.readI64(); - else: - iprot.skip(ftype) -+ elif fid == 4: -+ if ftype == TType.I16: -+ self.fetchType = iprot.readI16(); -+ else: -+ iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() -@@ -5185,6 +5193,10 @@ class TFetchResultsReq(object): - oprot.writeFieldBegin('maxRows', TType.I64, 3) - oprot.writeI64(self.maxRows) - oprot.writeFieldEnd() -+ if self.fetchType is not None: -+ oprot.writeFieldBegin('fetchType', TType.I16, 4) -+ oprot.writeI16(self.fetchType) -+ oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - -diff --git a/apps/beeswax/src/beeswax/api.py b/apps/beeswax/src/beeswax/api.py -index 91da7dd..f56b44a 100644 ---- a/apps/beeswax/src/beeswax/api.py -+++ b/apps/beeswax/src/beeswax/api.py -@@ -172,7 +172,8 @@ def watch_query_refresh_json(request, id): - handle, state = _get_query_handle_and_state(query_history) - - try: -- log = db.get_log(handle) -+ start_over = request.POST.get('log-start-over') == 'true' -+ log = db.get_log(handle, start_over=start_over) - except Exception, ex: - log = str(ex) - -diff --git a/apps/beeswax/src/beeswax/conf.py b/apps/beeswax/src/beeswax/conf.py -index 1667481..e7ffc8e 100644 ---- a/apps/beeswax/src/beeswax/conf.py -+++ b/apps/beeswax/src/beeswax/conf.py -@@ -60,6 +60,14 @@ SERVER_CONN_TIMEOUT = Config( - type=int, - help=_t('Timeout in seconds for Thrift calls.')) - -+USE_GET_LOG_API = Config( -+ key='use_get_log_api', -+ default=True, -+ type=coerce_bool, -+ help=_t('Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.' -+ 'If false, Hue will use the FetchResults() thrift call instead.') -+) -+ - BROWSE_PARTITIONED_TABLE_LIMIT = Config( - key='browse_partitioned_table_limit', - default=250, -diff --git a/apps/beeswax/src/beeswax/server/dbms.py b/apps/beeswax/src/beeswax/server/dbms.py -index baaba68..86b012b 100644 ---- a/apps/beeswax/src/beeswax/server/dbms.py -+++ b/apps/beeswax/src/beeswax/server/dbms.py -@@ -338,8 +338,8 @@ class HiveServer2Dbms(object): - return self.client.use(query) - - -- def get_log(self, query_handle): -- return self.client.get_log(query_handle) -+ def get_log(self, query_handle, start_over=True): -+ return self.client.get_log(query_handle, start_over) - - - def get_state(self, handle): -diff --git a/apps/beeswax/src/beeswax/server/hive_server2_lib.py b/apps/beeswax/src/beeswax/server/hive_server2_lib.py -index d7c222c..f7f2562 100644 ---- a/apps/beeswax/src/beeswax/server/hive_server2_lib.py -+++ b/apps/beeswax/src/beeswax/server/hive_server2_lib.py -@@ -14,6 +14,7 @@ - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. -+from itertools import imap - - import logging - import re -@@ -590,6 +591,15 @@ class HiveServerClient: - return res, schema - - -+ def fetch_log(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=1000): -+ req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation, -+ maxRows=max_rows, fetchType=1) -+ res = self.call(self._client.FetchResults, req) -+ -+ lines = imap(lambda r: r.colVals[0].stringVal.value, res.results.rows) -+ return '\n'.join(lines) -+ -+ - def get_operation_status(self, operation_handle): - req = TGetOperationStatusReq(operationHandle=operation_handle) - return self.call(self._client.GetOperationStatus, req) -@@ -759,9 +769,18 @@ class HiveServerClientCompatible(object): - return 'Does not exist in HS2' - - -- def get_log(self, handle): -+ def get_log(self, handle, start_over=True): - operationHandle = handle.get_rpc_handle() -- return self._client.get_log(operationHandle) -+ -+ if beeswax_conf.USE_GET_LOG_API.get() or self.query_server['server_name'] == 'impala': -+ return self._client.get_log(operationHandle) -+ else: -+ if start_over: -+ orientation = TFetchOrientation.FETCH_FIRST -+ else: -+ orientation = TFetchOrientation.FETCH_NEXT -+ -+ return self._client.fetch_log(operationHandle, orientation=orientation, max_rows=-1) - - - def get_databases(self): -diff --git a/apps/beeswax/src/beeswax/templates/execute.mako b/apps/beeswax/src/beeswax/templates/execute.mako -index 6b1f086..27909b0 100644 ---- a/apps/beeswax/src/beeswax/templates/execute.mako -+++ b/apps/beeswax/src/beeswax/templates/execute.mako -@@ -2637,6 +2637,11 @@ function setupCodeMirrorSubscription() { - - // Knockout - viewModel = new BeeswaxViewModel("${app_name}"); -+ -+% if not beeswax_conf.USE_GET_LOG_API.get() and app_name != 'impala': -+ viewModel.shouldAppendLogs = true; -+% endif -+ - % if query_history: - loadQueryHistory(${query_history.id}); - % elif design.id: -diff --git a/apps/beeswax/static/js/beeswax.vm.js b/apps/beeswax/static/js/beeswax.vm.js -index fdd2f78..6e6c65c 100644 ---- a/apps/beeswax/static/js/beeswax.vm.js -+++ b/apps/beeswax/static/js/beeswax.vm.js -@@ -560,7 +560,8 @@ function BeeswaxViewModel(server) { - self.watchQuery = function() { - var data = { - 'query-query': self.design.query.value(), -- 'query-database': self.database() -+ 'query-database': self.database(), -+ 'log-start-over': self.design.watch.logs().length == 0 - }; - $.extend(data, self.getSettingsFormData()); - $.extend(data, self.getFileResourcesFormData()); -@@ -589,6 +590,17 @@ function BeeswaxViewModel(server) { - $.ajax(request); - }; - -+ self.shouldAppendLogs = false; -+ -+ self.applyLogs = function(log) { -+ var lines = log.split("\n") -+ -+ if (self.shouldAppendLogs) { -+ lines = self.design.watch.logs().concat(lines); -+ } -+ self.design.watch.logs(lines); -+ }; -+ - self.watchQueryLoop = function(fn) { - var TIMEOUT = 100; - var timer = null; -@@ -607,7 +619,7 @@ function BeeswaxViewModel(server) { - clearTimeout(timer); - self.design.isRunning(false); - if (data.log) { -- self.design.watch.logs(data.log.split("\n")); -+ self.applyLogs(data.log) - // scroll logs - self.design.watch.jobUrls(data.jobUrls); - } -@@ -623,7 +635,7 @@ function BeeswaxViewModel(server) { - } else { - self.design.statement(data.statement); // In case new no result statement executed - if (data.log) { -- self.design.watch.logs(data.log.split("\n")); -+ self.applyLogs(data.log) - // scroll logs - self.design.watch.jobUrls(data.jobUrls); - } -diff --git a/apps/beeswax/thrift/TCLIService.thrift b/apps/beeswax/thrift/TCLIService.thrift -index 53ea3cf..63dad7e 100644 ---- a/apps/beeswax/thrift/TCLIService.thrift -+++ b/apps/beeswax/thrift/TCLIService.thrift -@@ -1002,6 +1002,9 @@ struct TFetchResultsReq { - // Max number of rows that should be returned in - // the rowset. - 3: required i64 maxRows -+ -+ // The type of a fetch results request. 0 represents Query output. 1 represents Log -+ 4: optional i16 fetchType = 0 - } - - struct TFetchResultsResp { -diff --git a/desktop/conf.dist/hue.ini b/desktop/conf.dist/hue.ini -index dc06a85..98c6bbb 100644 ---- a/desktop/conf.dist/hue.ini -+++ b/desktop/conf.dist/hue.ini -@@ -724,6 +724,10 @@ - # Timeout in seconds for thrift calls to Hive service - ## server_conn_timeout=120 - -+ # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. -+ # If false, Hue will use the FetchResults() thrift call instead. -+ ## use_get_log_api=true -+ - # Set a LIMIT clause when browsing a partitioned table. - # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. - ## browse_partitioned_table_limit=250 -diff --git a/desktop/conf/pseudo-distributed.ini.tmpl b/desktop/conf/pseudo-distributed.ini.tmpl -index 784c235..52f719b 100644 ---- a/desktop/conf/pseudo-distributed.ini.tmpl -+++ b/desktop/conf/pseudo-distributed.ini.tmpl -@@ -731,6 +731,10 @@ - # Timeout in seconds for thrift calls to Hive service - ## server_conn_timeout=120 - -+ # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. -+ # If false, Hue will use the FetchResults() thrift call instead. -+ ## use_get_log_api=true -+ - # Set a LIMIT clause when browsing a partitioned table. - # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. - ## browse_partitioned_table_limit=250 http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff b/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff deleted file mode 100644 index 36e1532..0000000 --- a/bigtop-packages/src/common/hue/patch1-bigtop-conf.diff +++ /dev/null @@ -1,24 +0,0 @@ -diff --git a/desktop/conf.dist/hue.ini~ b/desktop/conf.dist/hue.ini -index 9b30901..a2b9bea 100644 ---- a/desktop/conf.dist/hue.ini~ -+++ b/desktop/conf.dist/hue.ini -@@ -95,7 +95,8 @@ - - # Comma separated list of apps to not load at server startup. - # e.g.: pig,zookeeper -- ## app_blacklist= -+ # Bigtop does not bundle impala -+ app_blacklist=impala - - # The directory where to store the auditing logs. Auditing is disable if the value is empty. - # e.g. /var/log/hue/audit.log -@@ -722,7 +723,8 @@ - - # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. - # If false, Hue will use the FetchResults() thrift call instead. -- ## use_get_log_api=true -+ # Bigtop Hive only supports FetchResults() -+ use_get_log_api=false - - # Set a LIMIT clause when browsing a partitioned table. - # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop-packages/src/common/hue/patch2-disable-google-analytics.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hue/patch2-disable-google-analytics.diff b/bigtop-packages/src/common/hue/patch2-disable-google-analytics.diff deleted file mode 100644 index 83f7b37..0000000 --- a/bigtop-packages/src/common/hue/patch2-disable-google-analytics.diff +++ /dev/null @@ -1,13 +0,0 @@ ---- hue-3.7.1/desktop/conf.dist/hue.ini~ 2015-01-29 21:23:06.000000000 +0100 -+++ hue-3.7.1/desktop/conf.dist/hue.ini 2015-01-29 21:25:47.497503619 +0100 -@@ -79,8 +79,8 @@ - - # Help improve Hue with anonymous usage analytics. - # Use Google Analytics to see how many times an application or specific section of an application is used, nothing more. -- ## collect_usage=true -- -+ collect_usage=false -+ - # Support for HTTPS termination at the load-balancer level with SECURE_PROXY_SSL_HEADER. - ## secure_proxy_ssl_header=false - http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop-packages/src/deb/hue/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hue/rules b/bigtop-packages/src/deb/hue/rules index 4e4c8e6..2ff28c7 100755 --- a/bigtop-packages/src/deb/hue/rules +++ b/bigtop-packages/src/deb/hue/rules @@ -35,12 +35,6 @@ $(APPS:%=debian/hue-%.prerm): debian/hue-app.prerm.tpl dh $@ override_dh_auto_build: - # apply https://github.com/cloudera/hue/commit/2bbdd91e27cac9ab79e193dc4ad18d0647ae0048 - # see BIGTOP-1635 - rm -rf tools/virtual-bootstrap - rm -rf desktop/core/ext-py/ctypes-1.0.2/ctypes - tar -xvf debian/hue-virtualenv.tgz - # end of workaround env FULL_VERSION=${HUE_BASE_VERSION} bash -x debian/do-component-build -Dmaven.repo.local=${HOME}/.m2/repository override_dh_auto_install: build $(APPS:%=debian/hue-%.postinst) $(APPS:%=debian/hue-%.prerm) http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop-packages/src/rpm/hue/SPECS/hue.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/hue/SPECS/hue.spec b/bigtop-packages/src/rpm/hue/SPECS/hue.spec index 210eb7c..a77ec17 100644 --- a/bigtop-packages/src/rpm/hue/SPECS/hue.spec +++ b/bigtop-packages/src/rpm/hue/SPECS/hue.spec @@ -26,7 +26,6 @@ Source1: %{name}.init Source2: %{name}.init.suse Source3: do-component-build Source4: install_hue.sh -Source5: hue-virtualenv.tgz #BIGTOP_PATCH_FILES URL: http://github.com/cloudera/hue Requires: %{name}-common = %{version}-%{release} @@ -153,12 +152,6 @@ It supports a file browser, job tracker interface, cluster health monitor, and m #BIGTOP_PATCH_COMMANDS -#apply https://github.com/cloudera/hue/commit/2bbdd91e27cac9ab79e193dc4ad18d0647ae0048 -# see BIGTOP-1635 -rm -rf tools/virtual-bootstrap -rm -rf desktop/core/ext-py/ctypes-1.0.2/ctypes -tar -xvf %{SOURCE5} -#end of workaround ######################################## # Build ######################################## http://git-wip-us.apache.org/repos/asf/bigtop/blob/d4965671/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index 8866a05..564109b 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -208,8 +208,8 @@ $(eval $(call PACKAGE,giraph,GIRAPH)) HUE_NAME=hue HUE_RELNOTES_NAME=Hadoop User Experience HUE_PKG_NAME=hue -HUE_BASE_VERSION=3.7.1 -HUE_PKG_VERSION=3.7.1 +HUE_BASE_VERSION=3.8.1 +HUE_PKG_VERSION=3.8.1 HUE_RELEASE_VERSION=1 HUE_TARBALL_DST=hue-$(HUE_BASE_VERSION).tar.gz HUE_TARBALL_SRC=release-$(HUE_BASE_VERSION).tar.gz
