[ignite] branch master updated: Change file encoding to UTF-8 - Fixes #6465.
This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 0ad0950 Change file encoding to UTF-8 - Fixes #6465. 0ad0950 is described below commit 0ad095065f1b7045374e7b30b3220109ed29c620 Author: Alexandr Shapkin AuthorDate: Tue Apr 16 20:39:55 2019 +0300 Change file encoding to UTF-8 - Fixes #6465. Signed-off-by: Dmitriy Pavlov --- .../Config/Client/IgniteClientConfiguration.xml | Bin 3384 -> 1652 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Client/IgniteClientConfiguration.xml b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Client/IgniteClientConfiguration.xml index 89f9a69..291fcaf 100644 Binary files a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Client/IgniteClientConfiguration.xml and b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Client/IgniteClientConfiguration.xml differ
[ignite-teamcity-bot] branch ignite-11760 created (now dc808a8)
This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a change to branch ignite-11760 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git. at dc808a8 IGNITE-11760 Escaping Suite name if contains vertical line This branch includes the following new commits: new dc808a8 IGNITE-11760 Escaping Suite name if contains vertical line The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[ignite-teamcity-bot] 01/01: IGNITE-11760 Escaping Suite name if contains vertical line
This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-11760 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git commit dc808a881822e804c3e010516b2364bb2bf14bd0 Author: Dmitriy Pavlov AuthorDate: Tue Apr 16 20:38:47 2019 +0300 IGNITE-11760 Escaping Suite name if contains vertical line --- .../org/apache/ignite/ci/observer/BuildsInfo.java | 2 +- .../ignite/ci/tcbot/chain/PrChainsProcessor.java | 2 +- .../tcbot/visa/TcBotTriggerAndSignOffService.java | 29 -- .../apache/ignite/ci/tcbot/visa/VisaStatus.java| 2 +- .../ci/teamcity/ignited/ITeamcityIgnited.java | 2 +- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java index 9c1165d..82ef057 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java @@ -53,7 +53,7 @@ public class BuildsInfo { /** Server id. */ public final String srvId; -/** Build type id. */ +/** Build type id, for which build/observation/visa was ordered. */ public String buildTypeId; /** Branch name. */ diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java index 51f5266..bae99e1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/PrChainsProcessor.java @@ -171,7 +171,7 @@ public class PrChainsProcessor { } /** - * @param buildTypeId Suite name. + * @param buildTypeId Build type ID, for which visa was ordered. * @param branchForTc Branch for TeamCity. * @param srvId Server id. * @param prov Credentials. diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java index 1a5215f..2051015 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java @@ -725,7 +725,7 @@ public class TcBotTriggerAndSignOffService { * * @param srvId TC Server ID to take information about token from. * @param prov Credentials. - * @param buildTypeId Suite name. + * @param buildTypeId Build type ID, for which visa was ordered. * @param branchForTc Branch for TeamCity. * @param ticket JIRA ticket full name. E.g. IGNITE- * @return {@link Visa} instance. @@ -791,18 +791,20 @@ public class TcBotTriggerAndSignOffService { /** * @param suites Suite Current Status. * @param webUrl Build URL. + * @param buildTypeId Build type ID, for which visa was ordered. + * @param tcIgnited TC service. * @return Comment, which should be sent to the JIRA ticket. */ private String generateJiraComment(List suites, String webUrl, String buildTypeId, ITeamcityIgnited tcIgnited) { BuildTypeRefCompacted bt = tcIgnited.getBuildTypeRef(buildTypeId); -String suiteName = (bt != null ? bt.name(compactor) : buildTypeId); +String suiteNameUsedForVisa = (bt != null ? bt.name(compactor) : buildTypeId); StringBuilder res = new StringBuilder(); for (SuiteCurrentStatus suite : suites) { -res.append("{color:#d04437}").append(suite.name).append("{color}"); + res.append("{color:#d04437}").append(jiraEscText(suite.name)).append("{color}"); res.append(" [[tests ").append(suite.failedTests); if (suite.result != null && !suite.result.isEmpty()) @@ -814,9 +816,9 @@ public class TcBotTriggerAndSignOffService { res.append("* "); if (failure.suiteName != null && failure.testName != null) -res.append(failure.suiteName).append(": ").append(failure.testName); +res.append(failure.suiteName).append(": ").append(jiraEscText(failure.testName)); else -res.append(failure.name); +res.append(jiraEscText(failure.name)); FailureSummary recent = failure.histBaseBranch.recent; @@ -838,17 +840,28 @@ public class TcBotTriggerAndSignOffService { } if (res.length() > 0) { -res.insert(0, "{panel:title=" + suiteName + ": Possible Blockers|" + +res.insert(0, "{panel:title=" + jiraEscText(suiteNameUsedForVisa) + ":
[ignite] branch master updated: IGNITE-11759: [ML] Duplicate depenpecies for ml artifacts
This is an automated email from the ASF dual-hosted git repository. chief pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new dc992f5 IGNITE-11759: [ML] Duplicate depenpecies for ml artifacts dc992f5 is described below commit dc992f5611fa9d7c52dbc4e34cdce1de62dfaf9f Author: Alexey Platonov AuthorDate: Tue Apr 16 19:08:30 2019 +0300 IGNITE-11759: [ML] Duplicate depenpecies for ml artifacts This close #6461 --- modules/ml/mleap-model-parser/pom.xml | 6 +- modules/ml/spark-model-parser/pom.xml | 6 +- modules/ml/tensorflow-model-parser/pom.xml | 6 +- modules/ml/xgboost-model-parser/pom.xml| 6 +- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/modules/ml/mleap-model-parser/pom.xml b/modules/ml/mleap-model-parser/pom.xml index 4826bcf..a30dd0d 100644 --- a/modules/ml/mleap-model-parser/pom.xml +++ b/modules/ml/mleap-model-parser/pom.xml @@ -48,7 +48,11 @@ false - fastutil,h2 + + fastutil,h2,ignite-shmem,annotations,arpack_combined_all, + cache-api,commons-math3,commons-rng-client-api,commons-rng-core, +commons-rng-simple,core,ignite-ml + diff --git a/modules/ml/spark-model-parser/pom.xml b/modules/ml/spark-model-parser/pom.xml index 9dd60c8..c863917 100644 --- a/modules/ml/spark-model-parser/pom.xml +++ b/modules/ml/spark-model-parser/pom.xml @@ -48,7 +48,11 @@ false - fastutil,h2 + + fastutil,h2,ignite-shmem,annotations,arpack_combined_all, + cache-api,commons-math3,commons-rng-client-api,commons-rng-core, +commons-rng-simple,core,ignite-ml + diff --git a/modules/ml/tensorflow-model-parser/pom.xml b/modules/ml/tensorflow-model-parser/pom.xml index 551a60c..8002ff8 100644 --- a/modules/ml/tensorflow-model-parser/pom.xml +++ b/modules/ml/tensorflow-model-parser/pom.xml @@ -54,7 +54,11 @@ false - fastutil,h2 + + fastutil,h2,ignite-shmem,annotations,arpack_combined_all, + cache-api,commons-math3,commons-rng-client-api,commons-rng-core, +commons-rng-simple,core,ignite-ml + diff --git a/modules/ml/xgboost-model-parser/pom.xml b/modules/ml/xgboost-model-parser/pom.xml index c86b8d0..70f6465 100644 --- a/modules/ml/xgboost-model-parser/pom.xml +++ b/modules/ml/xgboost-model-parser/pom.xml @@ -52,7 +52,11 @@ false - fastutil,h2 + + fastutil,h2,ignite-shmem,annotations,arpack_combined_all, + cache-api,commons-math3,commons-rng-client-api,commons-rng-core, +commons-rng-simple,core,ignite-ml +
[ignite] branch master updated: IGNITE-11641 Fixed server node copies a lot of WAL files in WAL archive after restart
This is an automated email from the ASF dual-hosted git repository. dgovorukhin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 6670daf IGNITE-11641 Fixed server node copies a lot of WAL files in WAL archive after restart 6670daf is described below commit 6670daf79ae41be2adcf75765dd90fcbfa675e30 Author: Dmitriy Govorukhin AuthorDate: Tue Apr 16 18:01:42 2019 +0300 IGNITE-11641 Fixed server node copies a lot of WAL files in WAL archive after restart --- .../persistence/wal/FileWriteAheadLogManager.java | 167 +-- .../db/IgnitePdsStartWIthEmptyArchive.java | 178 + .../ignite/testsuites/IgnitePdsTestSuite4.java | 2 + 3 files changed, 301 insertions(+), 46 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index c6cdf7c..cf83e3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.persistence.wal; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.DataInput; import java.io.EOFException; import java.io.File; import java.io.FileFilter; @@ -29,6 +30,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.FileChannel; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.sql.Time; @@ -41,7 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.TreeSet; +import java.util.TreeMap; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongArray; @@ -126,6 +128,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.CREATE_NEW; import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE; @@ -137,8 +140,10 @@ import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED; import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_COMPACTED; import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR; import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_SUFFIX; import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory.LATEST_SERIALIZER_VERSION; import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE; +import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readPosition; import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader; /** @@ -445,22 +450,13 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl } }); -IgniteBiTuple tup = scanMinMaxArchiveIndices(); - segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled()); -segmentAware.lastTruncatedArchiveIdx(tup == null ? -1 : tup.get1() - 1); - -long lastAbsArchivedIdx = tup == null ? -1 : tup.get2(); - if (isArchiverEnabled()) -archiver = new FileArchiver(lastAbsArchivedIdx, log); +archiver = new FileArchiver(segmentAware, log); else archiver = null; -if (lastAbsArchivedIdx > 0) -segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx); - if (dsCfg.isWalCompactionEnabled()) { compressor = new FileCompressor(log); @@ -1098,41 +1094,36 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl } /** - * Lists files in archive directory and returns the indices of least and last archived files. - * In case of holes, first segment after last "hole" is considered as minimum. - * Example: minimum(0, 1, 10, 11, 20, 21, 22) should be 20 - * -
[ignite] branch master updated: IGNITE-11698 Fixed invalid classloader select for p2p - Fixes #6421.
This is an automated email from the ASF dual-hosted git repository. agoncharuk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 426e25a IGNITE-11698 Fixed invalid classloader select for p2p - Fixes #6421. 426e25a is described below commit 426e25aa4c8f71f0e0339ed9866775922d57901c Author: Vladislav Pyatkov AuthorDate: Tue Apr 16 15:48:58 2019 +0300 IGNITE-11698 Fixed invalid classloader select for p2p - Fixes #6421. Signed-off-by: Alexey Goncharuk --- .../cache/GridCacheDeploymentManager.java | 62 -- ...GridP2PComputeWithNestedEntryProcessorTest.java | 246 + .../ignite/testsuites/IgniteP2PSelfTestSuite.java | 2 + ...heDeploymenComputeWithNestedEntryProcessor.java | 77 +++ .../tests/p2p/pedicates/BinaryPredicate.java | 49 .../tests/p2p/pedicates/CompositePredicate.java| 53 + .../p2p/pedicates/FirstConsideredPredicate.java| 33 +++ .../p2p/pedicates/SecondConsideredPredicate.java | 33 +++ 8 files changed, 537 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java index 7bba1de..f0f2ed3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java @@ -92,6 +92,9 @@ public class GridCacheDeploymentManager extends GridCacheSharedManagerAdap /** */ private boolean depEnabled; +/** Class loader id for local thread. */ +private ThreadLocal localLdrId = new ThreadLocal<>(); + /** {@inheritDoc} */ @Override public void start0() throws IgniteCheckedException { globalLdr = new CacheClassLoader(cctx.gridConfig().getClassLoader()); @@ -365,6 +368,8 @@ public class GridCacheDeploymentManager extends GridCacheSharedManagerAdap DeploymentMode mode, Map participants ) { +localLdrId.set(ldrId); + assert depEnabled; if (mode == PRIVATE || mode == ISOLATED) { @@ -803,31 +808,25 @@ public class GridCacheDeploymentManager extends GridCacheSharedManagerAdap } } -for (CachedDeploymentInfo t : deps.values()) { -UUID sndId = t.senderId(); -IgniteUuid ldrId = t.loaderId(); -String userVer = t.userVersion(); -DeploymentMode mode = t.mode(); -Map participants = t.participants(); - -GridDeployment d = cctx.gridDeploy().getGlobalDeployment( -mode, -name, -name, -userVer, -sndId, -ldrId, -participants, -F.alwaysTrue()); +IgniteUuid curLdrId = localLdrId.get(); -if (d != null) { -Class cls = d.deployedClass(name); +if (curLdrId != null) { +CachedDeploymentInfo t = deps.get(curLdrId); + +if (t != null) { +Class cls = tryToloadClassFromCacheDep(name, t); if (cls != null) return cls; } } +for (CachedDeploymentInfo t : deps.values()) { +Class cls = tryToloadClassFromCacheDep(name, t); +if (cls != null) +return cls; +} + Class cls = getParent().loadClass(name); if (cls != null) @@ -837,6 +836,33 @@ public class GridCacheDeploymentManager extends GridCacheSharedManagerAdap } /** + * @param name Name of resource. + * @param deploymentInfo Grid cached deployment info. + * @return Class if can to load resource with the name or {@code null} otherwise. + */ +@Nullable private Class tryToloadClassFromCacheDep(String name, CachedDeploymentInfo deploymentInfo) { +UUID sndId = deploymentInfo.senderId(); +IgniteUuid ldrId = deploymentInfo.loaderId(); +String userVer = deploymentInfo.userVersion(); +DeploymentMode mode = deploymentInfo.mode(); +Map participants = deploymentInfo.participants(); + +GridDeployment d = cctx.gridDeploy().getGlobalDeployment( +mode, +name, +name, +userVer, +sndId, +ldrId, +participants, +F.alwaysTrue()); + +Class cls = d != null ? d.deployedClass(name) : null; + +return cls; +} + +
[ignite] branch master updated: IGNITE-11328: Ignite binary build is too big
This is an automated email from the ASF dual-hosted git repository. chief pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new 27f4fab IGNITE-11328: Ignite binary build is too big 27f4fab is described below commit 27f4fab7d4d56361bb77992563c8c710a99505fa Author: Alexey Platonov AuthorDate: Tue Apr 16 13:08:12 2019 +0300 IGNITE-11328: Ignite binary build is too big This closes #6144 --- examples/pom.xml | 6 ++ modules/ml/mleap-model-parser/pom.xml | 2 +- modules/ml/pom.xml | 13 - modules/ml/spark-model-parser/pom.xml | 1 + .../pom.xml| 22 ++ .../parser/TensorFlowBaseModelParser.java | 0 .../parser/TensorFlowGraphModelParser.java | 0 .../parser/TensorFlowSavedModelModelParser.java| 0 modules/ml/xgboost-model-parser/pom.xml| 1 + pom.xml| 1 + 10 files changed, 24 insertions(+), 22 deletions(-) diff --git a/examples/pom.xml b/examples/pom.xml index 3563d74..634fd8b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -96,6 +96,12 @@ +org.apache.ignite +ignite-ml-tensorflow-model-parser +${project.version} + + + commons-cli commons-cli 1.2 diff --git a/modules/ml/mleap-model-parser/pom.xml b/modules/ml/mleap-model-parser/pom.xml index 4221c3a..4826bcf 100644 --- a/modules/ml/mleap-model-parser/pom.xml +++ b/modules/ml/mleap-model-parser/pom.xml @@ -48,6 +48,7 @@ false + fastutil,h2 @@ -67,5 +68,4 @@ 0.13.0 - diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml index 478fb7a..bb66780 100644 --- a/modules/ml/pom.xml +++ b/modules/ml/pom.xml @@ -24,7 +24,6 @@ 4.0.0 3.6.1 -1.12.0 @@ -131,18 +130,6 @@ -org.tensorflow -tensorflow -${tensorflow.version} - - - -org.tensorflow -proto -${tensorflow.version} - - - org.mockito mockito-all ${mockito.version} diff --git a/modules/ml/spark-model-parser/pom.xml b/modules/ml/spark-model-parser/pom.xml index e852ac3..9dd60c8 100644 --- a/modules/ml/spark-model-parser/pom.xml +++ b/modules/ml/spark-model-parser/pom.xml @@ -48,6 +48,7 @@ false + fastutil,h2 diff --git a/modules/ml/spark-model-parser/pom.xml b/modules/ml/tensorflow-model-parser/pom.xml similarity index 81% copy from modules/ml/spark-model-parser/pom.xml copy to modules/ml/tensorflow-model-parser/pom.xml index e852ac3..551a60c 100644 --- a/modules/ml/spark-model-parser/pom.xml +++ b/modules/ml/tensorflow-model-parser/pom.xml @@ -22,8 +22,13 @@ http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> + 4.0.0 + +1.12.0 + + ignite-parent org.apache.ignite @@ -31,7 +36,8 @@ ../../../parent -ignite-ml-spark-model-parser + +ignite-ml-tensorflow-model-parser 2.8.0-SNAPSHOT http://ignite.apache.org @@ -48,6 +54,7 @@ false + fastutil,h2 @@ -63,16 +70,15 @@ -org.apache.parquet -parquet-hadoop -1.10.0 +org.tensorflow +tensorflow +${tensorflow.version} -org.apache.hadoop -hadoop-common -2.9.1 +org.tensorflow +proto +${tensorflow.version} - diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java b/modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java similarity index 100% rename from modules/ml/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java rename to modules/ml/tensorflow-model-parser/src/main/java/org/apache/ignite/ml/inference/parser/TensorFlowBaseModelParser.java diff --git