IGNITE-6120 Lazy query execution from Web console.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fea96f7f Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fea96f7f Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fea96f7f Branch: refs/heads/ignite-3478 Commit: fea96f7f86bc63b903d365439b762c48bc4db788 Parents: 74f8638 Author: Vasiliy Sisko <[email protected]> Authored: Tue Aug 22 17:51:55 2017 +0700 Committer: Andrey Novikov <[email protected]> Committed: Thu Sep 7 12:32:36 2017 +0700 ---------------------------------------------------------------------- .../internal/visor/query/VisorQueryTask.java | 1 + .../internal/visor/query/VisorQueryTaskArg.java | 41 +++++++++++++++++++- modules/web-console/backend/app/mongo.js | 5 ++- .../app/modules/agent/AgentManager.service.js | 18 +++++---- .../frontend/app/modules/sql/sql.controller.js | 21 ++++++++-- .../web-console/frontend/views/sql/sql.tpl.pug | 7 ++++ 6 files changed, 79 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/fea96f7f/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java index 9198d5e..c85ceea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java @@ -79,6 +79,7 @@ public class VisorQueryTask extends VisorOneNodeTask<VisorQueryTaskArg, VisorEit qry.setDistributedJoins(arg.isDistributedJoins()); qry.setEnforceJoinOrder(arg.isEnforceJoinOrder()); qry.setReplicatedOnly(arg.isReplicatedOnly()); + qry.setLazy(arg.getLazy()); long start = U.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/ignite/blob/fea96f7f/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java index 5013b0a..dd38332 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java @@ -52,6 +52,9 @@ public class VisorQueryTaskArg extends VisorDataTransferObject { /** Result batch size. */ private int pageSize; + /** Lazy query execution flag */ + private boolean lazy; + /** * Default constructor. */ @@ -68,8 +71,23 @@ public class VisorQueryTaskArg extends VisorDataTransferObject { * @param loc Flag whether to execute query locally. * @param pageSize Result batch size. */ - public VisorQueryTaskArg(String cacheName, String qryTxt, - boolean distributedJoins, boolean enforceJoinOrder, boolean replicatedOnly, boolean loc, int pageSize) { + public VisorQueryTaskArg(String cacheName, String qryTxt, boolean distributedJoins, + boolean enforceJoinOrder, boolean replicatedOnly, boolean loc, int pageSize) { + this(cacheName, qryTxt, distributedJoins, enforceJoinOrder, replicatedOnly, loc, pageSize, false); + } + + /** + * @param cacheName Cache name for query. + * @param qryTxt Query text. + * @param distributedJoins If {@code true} then distributed joins enabled. + * @param enforceJoinOrder If {@code true} then enforce join order. + * @param replicatedOnly {@code true} then query contains only replicated tables. + * @param loc Flag whether to execute query locally. + * @param pageSize Result batch size. + * @param lazy Lazy query execution flag. + */ + public VisorQueryTaskArg(String cacheName, String qryTxt, boolean distributedJoins, + boolean enforceJoinOrder, boolean replicatedOnly, boolean loc, int pageSize, boolean lazy) { this.cacheName = cacheName; this.qryTxt = qryTxt; this.distributedJoins = distributedJoins; @@ -77,6 +95,7 @@ public class VisorQueryTaskArg extends VisorDataTransferObject { this.replicatedOnly = replicatedOnly; this.loc = loc; this.pageSize = pageSize; + this.lazy = lazy; } /** @@ -128,6 +147,20 @@ public class VisorQueryTaskArg extends VisorDataTransferObject { return pageSize; } + /** + * Lazy query execution flag. + * + * @return Lazy flag. + */ + public boolean getLazy() { + return lazy; + } + + /** {@inheritDoc} */ + @Override public byte getProtocolVersion() { + return V2; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, cacheName); @@ -136,6 +169,7 @@ public class VisorQueryTaskArg extends VisorDataTransferObject { out.writeBoolean(enforceJoinOrder); out.writeBoolean(loc); out.writeInt(pageSize); + out.writeBoolean(lazy); } /** {@inheritDoc} */ @@ -146,6 +180,9 @@ public class VisorQueryTaskArg extends VisorDataTransferObject { enforceJoinOrder = in.readBoolean(); loc = in.readBoolean(); pageSize = in.readInt(); + + if (protoVer == V2) + lazy = in.readBoolean(); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/fea96f7f/modules/web-console/backend/app/mongo.js ---------------------------------------------------------------------- diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js index 28eaa5d..ecc833f 100644 --- a/modules/web-console/backend/app/mongo.js +++ b/modules/web-console/backend/app/mongo.js @@ -1041,7 +1041,10 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose value: Number, unit: Number }, - qryType: String + qryType: String, + nonCollocatedJoins: {type: Boolean, default: false}, + enforceJoinOrder: {type: Boolean, default: false}, + lazy: {type: Boolean, default: false} }] }); http://git-wip-us.apache.org/repos/asf/ignite/blob/fea96f7f/modules/web-console/frontend/app/modules/agent/AgentManager.service.js ---------------------------------------------------------------------- diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index 2b9d945..4ef274d 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -490,17 +490,21 @@ export default class IgniteAgentManager { * @param {Boolean} replicatedOnly Flag whether query contains only replicated tables. * @param {Boolean} local Flag whether to execute query locally. * @param {int} pageSz + * @param {Boolean} lazy query flag. * @returns {Promise} */ - querySql(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz) { + querySql(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy) { if (this.available('2.0.0')) { - return this.visorTask('querySqlX2', nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz) - .then(({error, result}) => { - if (_.isEmpty(error)) - return result; + const task = this.available('2.1.4') ? + this.visorTask('querySqlX2', nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy) : + this.visorTask('querySqlX2', nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz); - return Promise.reject(error); - }); + return task.then(({error, result}) => { + if (_.isEmpty(error)) + return result; + + return Promise.reject(error); + }); } cacheName = _.isEmpty(cacheName) ? null : cacheName; http://git-wip-us.apache.org/repos/asf/ignite/blob/fea96f7f/modules/web-console/frontend/app/modules/sql/sql.controller.js ---------------------------------------------------------------------- diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js index 30f53b7..ddc2fca 100644 --- a/modules/web-console/frontend/app/modules/sql/sql.controller.js +++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js @@ -30,6 +30,8 @@ const NON_COLLOCATED_JOINS_SINCE = '1.7.0'; const ENFORCE_JOIN_VERS = [['1.7.9', '1.8.0'], ['1.8.4', '1.9.0'], ['1.9.1']]; +const LAZY_QUERY_VERS = ['2.1.4']; + const _fullColName = (col) => { const res = []; @@ -1328,7 +1330,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', .then(() => _closeOldQuery(paragraph)) .then(() => args.localNid || _chooseNode(args.cacheName, false)) .then((nid) => agentMgr.querySql(nid, args.cacheName, args.query, args.nonCollocatedJoins, - args.enforceJoinOrder, false, !!args.localNid, args.pageSize)) + args.enforceJoinOrder, false, !!args.localNid, args.pageSize, args.lazy)) .then((res) => _processQueryResult(paragraph, false, res)) .catch((err) => paragraph.setError(err)); }; @@ -1370,9 +1372,19 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', return false; }; + $scope.lazyQueryAvailable = (paragraph) => { + const cache = _.find($scope.caches, {name: paragraph.cacheName}); + + if (cache) + return !!_.find(cache.nodes, (node) => Version.since(node.version, ...LAZY_QUERY_VERS)); + + return false; + }; + $scope.execute = (paragraph, local = false) => { const nonCollocatedJoins = !!paragraph.nonCollocatedJoins; const enforceJoinOrder = !!paragraph.enforceJoinOrder; + const lazy = !!paragraph.lazy; $scope.actionAvailable(paragraph, true) && _chooseNode(paragraph.cacheName, local) .then((nid) => { @@ -1393,14 +1405,15 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', maxPages: paragraph.maxPages, nonCollocatedJoins, enforceJoinOrder, - localNid: local ? nid : null + localNid: local ? nid : null, + lazy }; const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : paragraph.query; ActivitiesData.post({ action: '/queries/execute' }); - return agentMgr.querySql(nid, args.cacheName, qry, nonCollocatedJoins, enforceJoinOrder, false, local, args.pageSize); + return agentMgr.querySql(nid, args.cacheName, qry, nonCollocatedJoins, enforceJoinOrder, false, local, args.pageSize, lazy); }) .then((res) => { _processQueryResult(paragraph, true, res); @@ -1453,7 +1466,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', ActivitiesData.post({ action: '/queries/explain' }); - return agentMgr.querySql(nid, args.cacheName, args.query, false, !!paragraph.enforceJoinOrder, false, false, args.pageSize); + return agentMgr.querySql(nid, args.cacheName, args.query, false, !!paragraph.enforceJoinOrder, false, false, args.pageSize, false); }) .then((res) => _processQueryResult(paragraph, true, res)) .catch((err) => { http://git-wip-us.apache.org/repos/asf/ignite/blob/fea96f7f/modules/web-console/frontend/views/sql/sql.tpl.pug ---------------------------------------------------------------------- diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug index 0b8011a..1ef2a4c 100644 --- a/modules/web-console/frontend/views/sql/sql.tpl.pug +++ b/modules/web-console/frontend/views/sql/sql.tpl.pug @@ -114,6 +114,13 @@ mixin query-settings indexes are not selected in optimal order.' data-trigger='hover') input(type='checkbox' ng-model='paragraph.enforceJoinOrder') span Enforce join order + .row(ng-if='lazyQueryAvailable(paragraph)') + label.tipLabel(bs-tooltip data-placement='bottom' data-title='By default Ignite attempts to fetch the whole query result set to memory and send it to the client.<br/>\ + For small and medium result sets this provides optimal performance and minimize duration of internal database locks, thus increasing concurrency.<br/>\ + If result set is too big to fit in available memory this could lead to excessive GC pauses and even OutOfMemoryError.<br/>\ + Use this flag as a hint for Ignite to fetch result set lazily, thus minimizing memory consumption at the cost of moderate performance hit.' data-trigger='hover') + input(type='checkbox' ng-model='paragraph.lazy') + span Lazy result set mixin query-actions button.btn.btn-primary(ng-disabled='!actionAvailable(paragraph, true)' ng-click='execute(paragraph)') Execute
