Merge remote-tracking branch 'remotes/origin/master' into ignite-5075 # Conflicts: # modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java # modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5ef69009 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5ef69009 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5ef69009 Branch: refs/heads/ignite-5075 Commit: 5ef69009894cc2a6d33e31d923daed2f00bc45a1 Parents: 056b28f c488311 Author: sboikov <[email protected]> Authored: Thu Jun 1 11:52:35 2017 +0300 Committer: sboikov <[email protected]> Committed: Thu Jun 1 11:52:35 2017 +0300 ---------------------------------------------------------------------- DEVNOTES.txt | 74 ++-- assembly/dependencies-fabric-lgpl.xml | 2 + assembly/dependencies-fabric.xml | 2 + assembly/release-fabric-lgpl.xml | 1 + assembly/release-fabric.xml | 1 + assembly/release-yardstick.xml | 98 +++++ .../affinity/GridAffinityProcessor.java | 2 +- .../cache/CacheAffinitySharedManager.java | 35 +- .../processors/cache/GridCacheAdapter.java | 3 +- .../processors/cache/GridCacheProcessor.java | 39 +- .../CacheDataStructuresManager.java | 4 +- .../GridDhtPartitionsExchangeFuture.java | 2 +- .../closure/GridClosureProcessor.java | 11 +- .../service/GridServiceProcessor.java | 4 +- .../processors/service/GridServiceProxy.java | 4 +- .../processors/task/GridTaskProcessor.java | 7 +- .../task/GridTaskThreadContextKey.java | 5 +- .../ignite/spi/discovery/tcp/ClientImpl.java | 14 +- .../IgniteComputeTopologyExceptionTest.java | 2 +- ...IgniteClientCacheInitializationFailTest.java | 367 +++++++++++++++++++ .../discovery/AuthenticationRestartTest.java | 88 +++++ .../testsuites/IgniteCacheTestSuite4.java | 2 + .../IgniteSpiDiscoverySelfTestSuite.java | 2 + .../query/h2/sql/GridSqlQuerySplitter.java | 22 +- ...ClientReconnectCacheQueriesFailoverTest.java | 2 +- .../query/IgniteSqlSplitterSelfTest.java | 256 ++++++++++--- modules/ml/pom.xml | 13 + .../main/java/org/apache/ignite/ml/Model.java | 39 ++ .../ml/clustering/BaseKMeansClusterer.java | 98 +++++ .../apache/ignite/ml/clustering/Clusterer.java | 32 ++ .../ml/clustering/ClusterizationModel.java | 29 ++ .../clustering/KMeansDistributedClusterer.java | 298 +++++++++++++++ .../ml/clustering/KMeansLocalClusterer.java | 174 +++++++++ .../ignite/ml/clustering/KMeansModel.java | 79 ++++ .../ignite/ml/clustering/WeightedClusterer.java | 38 ++ .../ignite/ml/clustering/package-info.java | 22 ++ .../apache/ignite/ml/math/DistanceMeasure.java | 39 ++ .../ignite/ml/math/EuclideanDistance.java | 48 +++ .../org/apache/ignite/ml/math/MathUtils.java | 31 ++ .../java/org/apache/ignite/ml/math/Matrix.java | 2 + .../apache/ignite/ml/math/StorageConstants.java | 3 + .../org/apache/ignite/ml/math/VectorUtils.java | 41 +++ .../math/exceptions/ConvergenceException.java | 48 +++ .../exceptions/MathArithmeticException.java | 6 +- .../exceptions/MathIllegalNumberException.java | 51 +++ .../exceptions/MathIllegalStateException.java | 49 +++ .../exceptions/NumberIsTooSmallException.java | 79 ++++ .../ignite/ml/math/functions/Functions.java | 39 ++ .../apache/ignite/ml/math/impls/CacheUtils.java | 47 +++ .../ml/math/impls/matrix/AbstractMatrix.java | 2 + .../impls/matrix/SparseDistributedMatrix.java | 2 +- .../impls/storage/matrix/MapWrapperStorage.java | 93 +++++ .../vector/SparseLocalOnHeapVectorStorage.java | 32 ++ .../ml/math/impls/vector/DelegatingVector.java | 5 + .../ml/math/impls/vector/MapWrapperVector.java | 32 ++ .../ml/math/impls/vector/SparseLocalVector.java | 9 + .../ignite/ml/math/impls/vector/VectorView.java | 1 + .../ignite/ml/math/statistics/Variance.java | 53 +++ .../org/apache/ignite/ml/math/util/MapUtil.java | 38 ++ .../apache/ignite/ml/math/util/MatrixUtil.java | 38 +- .../org/apache/ignite/ml/IgniteMLTestSuite.java | 4 +- .../ml/clustering/ClusteringTesetSuite.java | 15 + .../KMeansDistributedClustererTest.java | 184 ++++++++++ .../ml/clustering/KMeansLocalClustererTest.java | 46 +++ .../apache/ignite/ml/clustering/KMeansUtil.java | 33 ++ .../ignite/ml/math/MathImplLocalTestSuite.java | 1 + modules/platforms/cpp/examples/Makefile.am | 1 + .../cpp/examples/compute-example/Makefile.am | 59 +++ .../compute-example/config/compute-example.xml | 52 +++ .../project/vs/compute-example.vcxproj | 102 ++++++ .../project/vs/compute-example.vcxproj.filters | 27 ++ .../compute-example/src/compute_example.cpp | 177 +++++++++ modules/platforms/cpp/examples/configure.ac | 1 + .../project/vs/continuous-query-example.vcxproj | 4 - .../src/continuous_query_example.cpp | 130 ++++++- .../examples/include/ignite/examples/address.h | 2 +- .../include/ignite/examples/organization.h | 4 +- .../cpp/examples/project/vs/ignite-examples.sln | 6 + .../Apache.Ignite.Core.nuspec | 5 +- modules/platforms/dotnet/build.ps1 | 5 +- modules/web-console/web-agent/pom.xml | 8 + parent/pom.xml | 8 +- pom.xml | 254 ++++--------- 83 files changed, 3452 insertions(+), 335 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 1c71dab,9063a12..c563b81 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@@ -378,37 -392,51 +378,44 @@@ public class CacheAffinitySharedManager nearCfg = req.nearCacheConfiguration(); } else { - startCache = cctx.cacheContext(action.descriptor().cacheId()) == null && - CU.affinityNode(cctx.localNode(), req.startCacheConfiguration().getNodeFilter()); + startCache = cctx.cacheContext(cacheDesc.cacheId()) == null && + CU.affinityNode(cctx.localNode(), cacheDesc.groupDescriptor().config().getNodeFilter()); } - if (startCache) { - cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion()); + try { + if (startCache) { + cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion()); - if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) { - if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty()) - U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName()); + if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) { - if (fut.discoCache().cacheAffinityNodes(req.cacheName()).isEmpty()) ++ if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty()) + U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName()); + } } + } ++ catch (IgniteCheckedException e) { ++ U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " + ++ "[cacheName=" + req.cacheName() + ']', e); + - if (!crd || !lateAffAssign) { - GridCacheContext cacheCtx = cctx.cacheContext(cacheDesc.cacheId()); - - if (cacheCtx != null && !cacheCtx.isLocal()) { - boolean clientCacheStarted = - req.clientStartOnly() && req.initiatingNodeId().equals(cctx.localNodeId()); ++ cctx.cache().forceCloseCache(fut.topologyVersion(), action, e); ++ } + } - Set<Integer> gprs = new HashSet<>(); - if (clientCacheStarted) - initAffinity(cacheDesc, cacheCtx.affinity().affinityCache(), fut, lateAffAssign); - else if (!req.clientStartOnly()) { - assert fut.topologyVersion().equals(cacheCtx.startTopologyVersion()); ++ Set<Integer> gprs = new HashSet<>(); - for (ExchangeActions.ActionData action : exchActions.newAndClientCachesStartRequests()) { - Integer grpId = action.descriptor().groupId(); - GridAffinityAssignmentCache aff = cacheCtx.affinity().affinityCache(); ++ for (ExchangeActions.ActionData action : exchActions.newAndClientCachesStartRequests()) { ++ Integer grpId = action.descriptor().groupId(); - if (gprs.add(grpId)) { - if (crd && lateAffAssign) - initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor()); - else { - CacheGroupInfrastructure grp = cctx.cache().cacheGroup(grpId); - assert aff.lastVersion().equals(AffinityTopologyVersion.NONE) : aff.lastVersion(); ++ if (gprs.add(grpId)) { ++ if (crd && lateAffAssign) ++ initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor());else { ++ CacheGroupInfrastructure grp = cctx.cache().cacheGroup(grpId); - if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.topologyVersion())) { - List<List<ClusterNode>> assignment = aff.calculate(fut.topologyVersion(), - fut.discoveryEvent(), fut.discoCache()); ++ if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.topologyVersion())) { + assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion(); - aff.initialize(fut.topologyVersion(), assignment); - } + initAffinity(registeredGrps.get(grp.groupId()), grp.affinity(), fut); } } - else - initStartedCacheOnCoordinator(fut, cacheDesc.cacheId()); - } - catch (IgniteCheckedException e) { - U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " + - "[cacheName=" + req.cacheName() + ']', e); - - cctx.cache().forceCloseCache(fut.topologyVersion(), action, e); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index cac8420,c7e8670..afb578b --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@@ -2031,10 -1889,9 +2031,10 @@@ public class GridCacheProcessor extend /** * @param req Stop request. + * @return Cache group for stopped cache. */ - private CacheGroupInfrastructure prepareCacheStop(DynamicCacheChangeRequest req) { - assert req.stop() || req.close() : req; - private void prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) { ++ private CacheGroupInfrastructure prepareCacheStop(DynamicCacheChangeRequest req, boolean forceClose) { + assert req.stop() || req.close() || forceClose : req; GridCacheAdapter<?, ?> cache = caches.remove(req.cacheName()); @@@ -2081,14 -1956,11 +2103,14 @@@ for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) { stopGateway(action.request()); - prepareCacheStop(action.request()); + prepareCacheStop(action.request(), forceClose); } - for (DynamicCacheChangeRequest req : exchActions.closeRequests(ctx.localNodeId())) { - String cacheName = req.cacheName(); + for (CacheGroupDescriptor grpDesc : exchActions.cacheGroupsToStop()) + stopCacheGroup(grpDesc.groupId()); + + for (ExchangeActions.ActionData req : exchActions.closeRequests(ctx.localNodeId())) { + String cacheName = req.request().cacheName(); IgniteCacheProxy<?, ?> proxy = jCacheProxies.get(cacheName); @@@ -2105,12 -1977,12 +2127,15 @@@ proxy.context().gate().onStopped(); - CacheGroupInfrastructure grp = prepareCacheStop(req.request()); - prepareCacheStop(req, forceClose); ++ CacheGroupInfrastructure grp = prepareCacheStop(req.request(), forceClose); + + if (grp != null && !grp.hasCaches()) + stopCacheGroup(grp.groupId()); } } + + if (forceClose) + completeCacheStartFuture(req, false, err); } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/5ef69009/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 65a20eb,2cd5629..d000e26 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@@ -1197,16 -1161,16 +1197,16 @@@ public class GridDhtPartitionsExchangeF boolean valid = true; - if (cacheCtx.config().getTopologyValidator() != null && !CU.isSystemCache(cacheCtx.name())) - valid = cacheCtx.config().getTopologyValidator().validate(discoEvt.topologyNodes()); + if (grp.topologyValidator() != null && !grp.systemCache()) + valid = grp.topologyValidator().validate(discoEvt.topologyNodes()); - m.put(cacheCtx.cacheId(), new CacheValidation(valid, lostParts)); + m.put(grp.groupId(), new CacheValidation(valid, lostParts)); } - cacheValidRes = m; + grpValidRes = m; } - cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err); + cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err, false); cctx.exchange().onExchangeDone(this, err);
