This is an automated email from the ASF dual-hosted git repository. sergeykamov pushed a commit to branch NLPCRAFT-70_NEW in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 41011383913c2fc6e113e7df09aa3f764414c93a Merge: 26fbef2 251364a Author: Sergey Kamov <[email protected]> AuthorDate: Tue Jul 27 16:29:20 2021 +0300 Merge branch 'master' into NLPCRAFT-70_NEW # Conflicts: # nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala # nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala # nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala # nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala hs_err_pid98656.log | 3856 ++++++++++++++++++++ .../apache/nlpcraft/examples/alarm/AlarmModel.java | 4 - .../alarm/src/main/resources/alarm_samples.txt | 2 +- .../src/main/resources/lightswitch_model.yaml | 2 + .../nlpcraft/example/minecraft/MinecraftModel.kt | 17 +- .../minecraft/src/main/resources/minecraft.yaml | 2 + .../sql/src/main/resources/sql_model.yaml | 2 + .../apache/nlpcraft/examples/time/TimeModel.java | 2 - .../nlpcraft/examples/weather/WeatherModel.java | 4 - .../org/apache/nlpcraft/common/NCService.scala | 17 +- .../nlpcraft/common/config/NCConfigurable.scala | 2 +- .../apache/nlpcraft/common/debug/NCLogHolder.scala | 5 +- .../org/apache/nlpcraft/common/util/NCUtils.scala | 4 + .../apache/nlpcraft/model/NCDialogFlowItem.java | 13 - .../org/apache/nlpcraft/model/NCIntentMatch.java | 15 - .../org/apache/nlpcraft/model/NCModelView.java | 5 + .../scala/org/apache/nlpcraft/model/NCToken.java | 9 + .../scala/org/apache/nlpcraft/model/NCVariant.java | 8 +- .../apache/nlpcraft/model/intent/NCIdlIntent.scala | 9 +- .../nlpcraft/model/intent/NCIdlIntentOptions.scala | 40 + .../apache/nlpcraft/model/intent/NCIdlTerm.scala | 2 +- .../model/intent/compiler/NCIdlCompiler.scala | 55 +- .../nlpcraft/model/intent/compiler/antlr4/NCIdl.g4 | 8 +- .../model/intent/compiler/antlr4/NCIdl.interp | 8 +- .../model/intent/compiler/antlr4/NCIdl.tokens | 4 +- .../intent/compiler/antlr4/NCIdlBaseListener.java | 18 +- .../model/intent/compiler/antlr4/NCIdlLexer.interp | 8 +- .../model/intent/compiler/antlr4/NCIdlLexer.java | 16 +- .../model/intent/compiler/antlr4/NCIdlLexer.tokens | 4 +- .../intent/compiler/antlr4/NCIdlListener.java | 22 +- .../model/intent/compiler/antlr4/NCIdlParser.java | 240 +- .../model/intent/solver/NCIntentSolver.scala | 1 - .../model/intent/solver/NCIntentSolverEngine.scala | 143 +- .../model/intent/solver/NCIntentSolverResult.scala | 6 +- .../nlpcraft/model/tools/cmdline/NCCli.scala | 114 +- .../nlpcraft/model/tools/cmdline/NCCliBase.scala | 9 +- .../model/tools/cmdline/NCCliCommands.scala | 150 +- .../model/tools/cmdline/NCCliRestSpec.scala | 19 + .../model/tools/test/NCTestClientBuilder.java | 22 +- .../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 247 +- .../probe/mgrs/conn/NCConnectionManager.scala | 7 +- .../probe/mgrs/conversation/NCConversation.scala | 48 +- .../probe/mgrs/deploy/NCDeployManager.scala | 8 +- .../mgrs/dialogflow/NCDialogFlowManager.scala | 58 +- .../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 12 +- .../mgrs/nlp/enrichers/model/NCModelEnricher.scala | 18 +- .../server/ignite/NCIgniteCacheStore.scala | 2 +- .../nlpcraft/server/mdo/NCProbeModelMdo.scala | 1 + .../opencensus/NCOpenCensusServerStats.scala | 7 +- .../nlpcraft/server/probe/NCProbeManager.scala | 162 +- .../nlpcraft/server/query/NCQueryManager.scala | 2 +- .../nlpcraft/server/rest/NCBasicRestApi.scala | 229 +- .../server/sugsyn/NCSuggestSynonymManager.scala | 38 +- .../nlpcraft/server/user/NCUserManager.scala | 2 +- nlpcraft/src/test/resources/log4j2.xml | 2 +- .../scala/org/apache/nlpcraft/NCTestContext.scala | 7 +- .../scala/org/apache/nlpcraft/NCTestElement.scala | 20 +- .../nlpcraft/model/NCTrueIntentIdlSpec.scala | 2 +- .../intent/idl/compiler/NCIdlCompilerSpec.scala | 71 +- .../model/intent/idl/options/NCFreeWordsSpec.scala | 75 + .../model/intent/idl/options/NCOrderedSpec.scala | 75 + .../model/intent/idl/options/NCStmOnlySpec.scala | 130 + .../model/intent/idl/options/NCSysTokensSpec.scala | 75 + .../intent/idl/options/NCUserTokensSpec.scala | 75 + .../model/properties/NCTokensPropertiesSpec.scala | 68 +- .../mgrs/nlp/enrichers/NCEnricherBaseSpec.scala | 4 +- .../nlpcraft/server/rest/NCRestModelSpec.scala | 75 +- .../apache/nlpcraft/server/rest/NCRestSpec.scala | 1 - .../nlpcraft/server/rest/RestTestModel.scala | 5 +- openapi/nlpcraft_swagger.yml | 128 +- 70 files changed, 5912 insertions(+), 609 deletions(-) diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala index cdcb7a4,d2a4619..6a7cffb --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/conn/NCConnectionManager.scala @@@ -31,11 -30,11 +30,10 @@@ import org.apache.nlpcraft.probe.mgrs.m import java.io.{EOFException, IOException, InterruptedIOException} import java.net.{InetAddress, NetworkInterface} - import java.{lang, util} + import java.util import java.util.concurrent.CountDownLatch - import java.util.{Collections, Properties, TimeZone} + import java.util.{Properties, TimeZone} import scala.collection.mutable - import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, SeqHasAsJava, SetHasAsJava, SetHasAsScala} -import scala.jdk.CollectionConverters.{SetHasAsJava, SetHasAsScala} /** * Probe down/up link connection manager. @@@ -269,9 -223,7 +267,10 @@@ object NCConnectionManager extends NCSe mdl.getName, mdl.getVersion, new util.HashSet[String](mdl.getEnabledBuiltInTokens), - new util.HashSet[String](mdl.getElements.asScala.map(_.getId).asJava) ++ new util.HashSet[String](mdl.getElements.asScala.map(_.getId).asJava), + singleValues, + corpus, + categoriesElements ) }) ), cryptoKey) diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala index 8a538ab,1b6001b..a727912 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/mdo/NCProbeModelMdo.scala @@@ -36,7 -28,7 +36,8 @@@ case class NCProbeModelMdo @NCMdoField name: String, @NCMdoField version: String, @NCMdoField enabledBuiltInTokens: Set[String], - @NCMdoField elementIds: Set[String] ++ @NCMdoField elementIds: Set[String], + @NCMdoField ctxWordConfig: Option[NCCtxWordCategoriesConfigMdo] ) extends NCAnnotatedMdo[NCProbeModelMdo] { override def hashCode(): Int = s"$id$name".hashCode() diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala index a18ad80,92238a2..3abd5cb --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala @@@ -619,9 -621,7 +626,10 @@@ object NCProbeManager extends NCServic String, String, java.util.Set[String], - java.util.Set[String] ++ java.util.Set[String], + java.util.Map[String, java.util.Map[String, java.util.Set[String]]], + java.util.Set[String], + java.util.Map[String, Double] )]]("PROBE_MODELS"). map { case ( @@@ -629,41 -629,20 +637,44 @@@ mdlName, mdlVer, enabledBuiltInToks, - elmIds ++ elmIds, + singleValues, + corpus, + categoriesElements ) => require(mdlId != null) require(mdlName != null) require(mdlVer != null) require(enabledBuiltInToks != null) + require(elmIds != null) + require(singleValues.isEmpty && corpus.isEmpty || !singleValues.isEmpty && !corpus.isEmpty) NCProbeModelMdo( id = mdlId, name = mdlName, version = mdlVer, enabledBuiltInTokens = enabledBuiltInToks.asScala.toSet, - elementIds = elmIds.asScala.toSet ++ elementIds = elmIds.asScala.toSet, + ctxWordConfig = + if (!singleValues.isEmpty) { + Some( + NCCtxWordCategoriesConfigMdo( + probeId = probeId, + modelId = mdlId, + singleValues = singleValues.asScala.map { + case (elemId, map) => + elemId -> + map.asScala.map { + case (value, syns) => value -> syns.asScala.toSet + }.toMap + }.toMap, + corpus = corpus.asScala.toSet, + elements = categoriesElements.asScala.toMap + ) + ) + } + else + None ) }.toSet @@@ -714,7 -693,24 +725,24 @@@ } } } - + + /** + * + * @param probeMsg + * @param m + */ + private def processJavaMetaMessage(probeMsg: NCProbeMessage, m: ConcurrentHashMap[String, Promise[JavaMeta]]): Unit = { + val p = m.remove(probeMsg.data[String]("reqGuid")) + + if (p != null) + probeMsg.dataOpt[String]("resp") match { + case Some(resp) => p.success(GSON.fromJson(resp, TYPE_JAVA_META)) + case None => p.failure(new NCE(probeMsg.data[String]("error"))) + } + else + logger.warn(s"Message ignored: $probeMsg") + } - ++ /** * Processes the messages received from the probe. * diff --cc nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala index e9d6b43,a12a4e8..301bc59 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala @@@ -270,9 -270,9 +270,9 @@@ object NCQueryManager extends NCServic tbl += (s"${b("Server Request ID")}", m(srvReqId)) tbl += (s"${b("Data")}", U.prettyJson(data.orNull).split("\n").toSeq) - logger.info(s"New request received:\n$tbl") + logger.info(s"New user request received:\n$tbl") - val enabledBuiltInToks = NCProbeManager.getModel(mdlId, span).enabledBuiltInTokens + val mdl = NCProbeManager.getModel(mdlId, span) @throws[NCE] def unzipProperties(gzipOpt: Option[String]): Option[JavaMeta] =
