This is an automated email from the ASF dual-hosted git repository. aradzinski pushed a commit to branch NLPCRAFT-296 in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 679db5af91174d3e2a454df8cb1c9533a530c284 Author: Sergey Kamov <[email protected]> AuthorDate: Sun May 30 14:14:41 2021 +0300 WIP. --- nlpcraft/pom.xml | 8 +++---- .../org/apache/nlpcraft/common/util/NCUtils.scala | 2 +- .../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 2 +- .../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 2 +- .../probe/mgrs/sentence/NCSentenceManager.scala | 10 +++++--- .../enrichers/stopword/NCStopWordEnricher.scala | 27 ++++++++++++++-------- .../server/sugsyn/NCSuggestSynonymManager.scala | 4 ++-- .../nlpcraft/common/ascii/NCAsciiTableSpec.scala | 1 + .../compiler/functions/NCIdlFunctionsDate.scala | 1 + .../apache/nlpcraft/server/rest/NCRestSpec.scala | 2 +- pom.xml | 20 ++++++++-------- 11 files changed, 47 insertions(+), 32 deletions(-) diff --git a/nlpcraft/pom.xml b/nlpcraft/pom.xml index 779a4ba..4d8c292 100644 --- a/nlpcraft/pom.xml +++ b/nlpcraft/pom.xml @@ -190,15 +190,15 @@ </dependency> <dependency> <groupId>com.typesafe.akka</groupId> - <artifactId>akka-http_2.12</artifactId> + <artifactId>akka-http_2.13</artifactId> </dependency> <dependency> <groupId>com.typesafe.akka</groupId> - <artifactId>akka-stream_2.12</artifactId> + <artifactId>akka-stream_2.13</artifactId> </dependency> <dependency> <groupId>com.typesafe.akka</groupId> - <artifactId>akka-http-spray-json_2.12</artifactId> + <artifactId>akka-http-spray-json_2.13</artifactId> </dependency> <dependency> <groupId>com.fasterxml.jackson.dataformat</groupId> @@ -210,7 +210,7 @@ </dependency> <dependency> <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-scala_2.12</artifactId> + <artifactId>jackson-module-scala_2.13</artifactId> </dependency> <dependency> <groupId>com.google.cloud</groupId> diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala index 3e7b046..c698e10 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala @@ -1016,7 +1016,7 @@ object NCUtils extends LazyLogging { */ def asFuture[T]( body: Unit => T, - onFailure: Throwable => Unit = _ => Unit, + onFailure: Throwable => Unit = _ => (), onSuccess: T => Unit = (_: T) => ())(implicit ec: ExecutionContext): Future[T] = { val fut = Future { body(()) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala index 8a88008..1f96350 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala @@ -101,7 +101,7 @@ object NCCommandManager extends NCService { txt = msg.data[String]("txt"), nlpSens = msg.data[JList[NCNlpSentence]]("nlpSens").asScala, usrId = msg.data[Long]("userId"), - senMeta = msg.data[java.util.Map[String, JSerializable]]("senMeta").asScala, + senMeta = msg.data[java.util.Map[String, JSerializable]]("senMeta").asScala.toMap, mdlId = msg.data[String]("mdlId"), enableLog = msg.data[Boolean]("enableLog"), span diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala index 6b92656..1b10224 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/model/NCModelEnricher.scala @@ -286,7 +286,7 @@ object NCModelEnricher extends NCProbeEnricher { elem = mdl.elements.getOrElse(elemId, throw new NCE(s"Custom model parser returned unknown element ID: $elemId")), toks = matchedToks, direct = true, - metaOpt = Some(e.getMetadata.asScala) + metaOpt = Some(e.getMetadata.asScala.toMap) ) }) } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala index 4c49d1d..63913ac 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala @@ -28,6 +28,7 @@ import java.io.{Serializable => JSerializable} import java.util import java.util.{List => JList} import scala.collection.mutable +import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable import scala.jdk.CollectionConverters.{ListHasAsScala, SeqHasAsJava, SetHasAsJava} import scala.language.implicitConversions @@ -655,7 +656,7 @@ object NCSentenceManager extends NCService { distinct // Optimization. Deletes all wholly swallowed notes. - val links = getLinks(sen.flatten) + val links = getLinks(sen.tokens.flatMap(p => p)) val swallowed = delCombs. @@ -690,13 +691,16 @@ object NCSentenceManager extends NCService { groupBy { case (idx, _) => idx }. map { case (_, seq) => seq.map { case (_, note) => note }.toSet }. toSeq.sortBy(-_.size) + + def findCombinations(): Seq[Seq[NCNlpSentenceNote]] = + NCSentenceHelper.findCombinations(toksByIdx.map(_.asJava).asJava, pool).asScala.map(_.asScala.toSeq) val seqSens = combCache. getOrElseUpdate(sen.srvReqId, mutable.HashMap.empty[CacheKey, CacheValue]). getOrElseUpdate( toksByIdx, - NCSentenceHelper.findCombinations(toksByIdx.map(_.asJava).asJava, pool).asScala.map(_.asScala) + findCombinations() ).par. flatMap(delComb => { val nsClone = sen.clone() @@ -721,7 +725,7 @@ object NCSentenceManager extends NCService { Holder( // We have to delete some keys to have possibility to compare sentences. - notes.map(_.clone().filter { case (name, _) => name != "direct" }), + notes.map(_.clone().filter { case (name, _) => name != "direct" }).toSeq, sen, notes.filter(_.isNlp).map(p => if (p.isDirect) 0 else 1).sum ) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala index 0762677..725a737 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordEnricher.scala @@ -232,15 +232,24 @@ object NCStopWordEnricher extends NCServerEnricher { } } - def mkEntry[T](f: WordForm, mkT: Unit => T, isExc: Boolean):((Boolean, WordForm), T) = (isExc, f) -> mkT(()) - def mkMap[T](mkT: Unit => T): Map[(Boolean, WordForm), T] = - WordForm - .values - .toSet - .flatMap(f => Map( - mkEntry(f, mkT, isExc = true), - mkEntry(f, mkT, isExc = false) - )).toMap + type Key = (Boolean, WordForm) + def mkMap[T](mkT: Unit => T): Map[Key, T] = { + val m = mutable.Map.empty[Key, T] + + def add(f: WordForm, mkT: Unit => T, isExc: Boolean): Unit = { + val tuple: (Key, T) = (isExc, f) -> mkT(()) + + m += tuple._1 → tuple._2 + } + + WordForm.values.foreach(f => { + add(f, mkT, isExc = true) + add(f, mkT, isExc = false) + + }) + + m.toMap + } // Prepares collections. val mHash = mkMap(_ => new Condition[Word]()) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala index cd3777b..47f8171 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala @@ -38,7 +38,7 @@ import java.util.concurrent._ import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future, Promise} -import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsScala, SeqHasAsJava} +import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success} /** @@ -411,7 +411,7 @@ object NCSuggestSynonymManager extends NCService { m }).asJava - }.asJava + }.toMap.asJava promise.success( NCSuggestSynonymResult( diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala index a3bf00f..f914eaa 100644 --- a/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/common/ascii/NCAsciiTableSpec.scala @@ -17,6 +17,7 @@ package org.apache.nlpcraft.common.ascii +import org.apache.nlpcraft.common.U import org.apache.nlpcraft.common.ansi.NCAnsi._ import org.junit.jupiter.api.Test diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala index 0bac73e..97a149b 100644 --- a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/intent/idl/compiler/functions/NCIdlFunctionsDate.scala @@ -17,6 +17,7 @@ package org.apache.nlpcraft.model.intent.idl.compiler.functions +import org.apache.nlpcraft.common.U import org.junit.jupiter.api.Test import java.time.temporal.IsoFields diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala index 948e5ee..03bf811 100644 --- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestSpec.scala @@ -130,7 +130,7 @@ import org.apache.nlpcraft.server.rest.NCRestSpec._ class NCRestSpec extends NCTestContext { type ResponseContent = java.util.Map[String, Object] type ResponseList = JList[ResponseContent] - type JList[T] = JList[T] + type JList[T] = java.util.List[T] protected var tkn: String = _ /** diff --git a/pom.xml b/pom.xml index bf77506..be9d907 100644 --- a/pom.xml +++ b/pom.xml @@ -128,9 +128,9 @@ <maven.clean.plugin.ver>3.1.0</maven.clean.plugin.ver> <apache.rat.plugin.ver>0.13</apache.rat.plugin.ver> <org.antlr4.ver>4.9</org.antlr4.ver> - <akka.http.2.12.ver>10.2.0</akka.http.2.12.ver> - <akka.stream.2.12.ver>2.6.8</akka.stream.2.12.ver> - <akka.http.spray.json.2.12.ver>10.2.0</akka.http.spray.json.2.12.ver> + <akka.http.2.13.ver>10.2.4</akka.http.2.13.ver> + <akka.stream.2.13.ver>2.6.14</akka.stream.2.13.ver> + <akka.http.spray.json.2.13.ver>10.2.4</akka.http.spray.json.2.13.ver> <google.guava.ver>29.0-jre</google.guava.ver> <diff.utils.ver>4.7</diff.utils.ver> <vertical.blank.ver>1.0.1</vertical.blank.ver> @@ -409,20 +409,20 @@ <dependency> <groupId>com.typesafe.akka</groupId> - <artifactId>akka-http_2.12</artifactId> - <version>${akka.http.2.12.ver}</version> + <artifactId>akka-http_2.13</artifactId> + <version>${akka.http.2.13.ver}</version> </dependency> <dependency> <groupId>com.typesafe.akka</groupId> - <artifactId>akka-stream_2.12</artifactId> - <version>${akka.stream.2.12.ver}</version> + <artifactId>akka-stream_2.13</artifactId> + <version>${akka.stream.2.13.ver}</version> </dependency> <dependency> <groupId>com.typesafe.akka</groupId> - <artifactId>akka-http-spray-json_2.12</artifactId> - <version>${akka.http.spray.json.2.12.ver}</version> + <artifactId>akka-http-spray-json_2.13</artifactId> + <version>${akka.http.spray.json.2.13.ver}</version> </dependency> <dependency> @@ -439,7 +439,7 @@ <dependency> <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-scala_2.12</artifactId> + <artifactId>jackson-module-scala_2.13</artifactId> <version>${jackson.ver}</version> </dependency>
