This is an automated email from the ASF dual-hosted git repository. sergeykamov pushed a commit to branch NLPCRAFT-379 in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 7af50af47614b71ffd7754eaca9d9accfbc9882b Author: Sergey Kamov <[email protected]> AuthorDate: Sun Jul 25 11:53:12 2021 +0300 WIP. --- .../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 127 ++++++++++++++++++--- .../nlpcraft/server/probe/NCProbeManager.scala | 76 ++++++------ .../nlpcraft/server/rest/NCBasicRestApi.scala | 46 ++++++++ .../server/sugsyn/NCSuggestSynonymManager.scala | 2 +- .../nlpcraft/server/rest/NCRestModelSpec.scala | 11 ++ 5 files changed, 213 insertions(+), 49 deletions(-) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala index f63a088..26d29c1 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala @@ -17,11 +17,12 @@ package org.apache.nlpcraft.probe.mgrs.cmd -import com.google.gson.Gson +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule import io.opencensus.trace.Span import org.apache.nlpcraft.common.nlp.NCNlpSentence import org.apache.nlpcraft.common.{NCService, _} -import org.apache.nlpcraft.model.NCToken +import org.apache.nlpcraft.model.{NCCustomParser, NCElement, NCModelView, NCToken, NCValue, NCValueLoader} import org.apache.nlpcraft.probe.mgrs.NCProbeMessage import org.apache.nlpcraft.probe.mgrs.conn.NCConnectionManager import org.apache.nlpcraft.probe.mgrs.conversation.NCConversationManager @@ -30,21 +31,18 @@ import org.apache.nlpcraft.probe.mgrs.model.NCModelManager import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnrichmentManager import java.io.{Serializable => JSerializable} -import java.util -import java.util.{Collections, List => JList} -import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala} +import java.util.{Collections, Optional, List => JList} +import java.{lang, util} +import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsJava, SetHasAsScala} /** * Probe commands processor. */ object NCCommandManager extends NCService { - private final val GSON = new Gson() + private final val JS_MAPPER = new ObjectMapper() + + JS_MAPPER.registerModule(DefaultScalaModule) - /** - * Starts this service. - * - * @param parent Optional parent span. - */ override def start(parent: Span): NCService = startScopedSpan("start", parent) { _ => ackStarting() ackStarted() @@ -128,7 +126,7 @@ object NCCommandManager extends NCService { span ) - case "S2P_MODEL_INFO" => + case "S2P_MODEL_SYNS_INFO" => send0( mkMsg = () => { val mdlId = msg.data[String]("mdlId") @@ -142,9 +140,9 @@ object NCCommandManager extends NCService { mdlData.samples.map(p => p._1 -> p._2.map(_.asJava).asJava).toMap.asJava NCProbeMessage( - "P2S_MODEL_INFO", + "P2S_MODEL_SYNS_INFO", "reqGuid" -> msg.getGuid, - "resp" -> GSON.toJson( + "resp" -> JS_MAPPER.writeValueAsString( Map( "macros" -> macros.asInstanceOf[JSerializable], "synonyms" -> syns.asInstanceOf[JSerializable], @@ -155,7 +153,7 @@ object NCCommandManager extends NCService { }, mkErrorMsg = e => NCProbeMessage( - "P2S_MODEL_INFO", + "P2S_MODEL_SYNS_INFO", "reqGuid" -> msg.getGuid, "error" -> e.getLocalizedMessage ), @@ -182,7 +180,7 @@ object NCCommandManager extends NCService { NCProbeMessage( "P2S_MODEL_ELEMENT_INFO", "reqGuid" -> msg.getGuid, - "resp" -> GSON.toJson( + "resp" -> JS_MAPPER.writeValueAsString( Map( "synonyms" -> elm.getSynonyms.asInstanceOf[JSerializable], "values" -> vals.asInstanceOf[JSerializable], @@ -200,6 +198,103 @@ object NCCommandManager extends NCService { span ) + case "S2P_MODEL_INFO" => + send0( + mkMsg = () => { + val mdlId = msg.data[String]("mdlId") + + val mdl = NCModelManager.getModel(mdlId).model + + val convertedMdl = + new NCModelView { + // As is. + override def getId: String = mdl.getId + override def getName: String = mdl.getName + override def getVersion: String = mdl.getVersion + override def getDescription: String = mdl.getDescription + override def getOrigin: String = mdl.getOrigin + override def getMaxUnknownWords: Int = mdl.getMaxUnknownWords + override def getMaxFreeWords: Int = mdl.getMaxFreeWords + override def getMaxSuspiciousWords: Int = mdl.getMaxSuspiciousWords + override def getMinWords: Int = mdl.getMinWords + override def getMaxWords: Int = mdl.getMaxWords + override def getMinTokens: Int = mdl.getMinTokens + override def getMaxTokens: Int = mdl.getMaxTokens + override def getMinNonStopwords: Int = mdl.getMinNonStopwords + override def isNonEnglishAllowed: Boolean = mdl.isNonEnglishAllowed + override def isNotLatinCharsetAllowed: Boolean = mdl.isNotLatinCharsetAllowed + override def isSwearWordsAllowed: Boolean = mdl.isSwearWordsAllowed + override def isNoNounsAllowed: Boolean = mdl.isNoNounsAllowed + override def isPermutateSynonyms: Boolean = mdl.isPermutateSynonyms + override def isDupSynonymsAllowed: Boolean = mdl.isDupSynonymsAllowed + override def getMaxTotalSynonyms: Int = mdl.getMaxTotalSynonyms + override def isNoUserTokensAllowed: Boolean = mdl.isNoUserTokensAllowed + override def isSparse: Boolean = mdl.isSparse + override def getMetadata: util.Map[String, AnyRef] = mdl.getMetadata + override def getAdditionalStopWords: util.Set[String] = + mdl.getAdditionalStopWords + override def getExcludedStopWords: util.Set[String] = mdl.getExcludedStopWords + override def getSuspiciousWords: util.Set[String] = mdl.getSuspiciousWords + override def getMacros: util.Map[String, String] = mdl.getMacros + override def getEnabledBuiltInTokens: util.Set[String] = + mdl.getEnabledBuiltInTokens + override def getAbstractTokens: util.Set[String] = mdl.getAbstractTokens + override def getMaxElementSynonyms: Int = mdl.getMaxElementSynonyms + override def isMaxSynonymsThresholdError: Boolean = + mdl.isMaxSynonymsThresholdError + override def getConversationTimeout: Long = mdl.getConversationTimeout + override def getConversationDepth: Int = mdl.getConversationDepth + override def getRestrictedCombinations: util.Map[String, util.Set[String]] = + mdl.getRestrictedCombinations + + // Cleared. + override def getParsers: JList[NCCustomParser] = null + // Converted. + override def getElements: util.Set[NCElement] = mdl.getElements.asScala.map(e => + new NCElement { + // As is. + override def getId: String = e.getId + override def getGroups: JList[String] = e.getGroups + override def getMetadata: util.Map[String, AnyRef] = e.getMetadata + override def getDescription: String = e.getDescription + override def getParentId: String = e.getParentId + override def getSynonyms: JList[String] = e.getSynonyms + override def isPermutateSynonyms: Optional[lang.Boolean] = + e.isPermutateSynonyms + override def isSparse: Optional[lang.Boolean] = e.isSparse + + // Cleared. + override def getValueLoader: Optional[NCValueLoader] = null + // Converted. + override def getValues: JList[NCValue] = + if (e.getValues != null) { + e.getValues.asScala.map(v => new NCValue { + override def getName: String = v.getName + // Cleared. + override def getSynonyms: JList[String] = null + }).asJava + } + else + null + } + ).asJava + } + + NCProbeMessage( + "P2S_MODEL_INFO", + "reqGuid" -> msg.getGuid, + "resp" -> JS_MAPPER.writeValueAsString(convertedMdl) + ) + }, + mkErrorMsg = e => + NCProbeMessage( + "P2S_MODEL_SYNS_INFO", + "reqGuid" -> msg.getGuid, + "error" -> e.getLocalizedMessage + ), + span + ) + case _ => logger.error(s"Received unknown server message (you need to update the probe): ${msg.getType}") } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala index 04792f6..3c7c46c 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala @@ -25,7 +25,7 @@ import org.apache.nlpcraft.common.config.NCConfigurable import org.apache.nlpcraft.common.crypto.NCCipher import org.apache.nlpcraft.common.makro.NCMacroParser import org.apache.nlpcraft.common.nlp.NCNlpSentence -import org.apache.nlpcraft.common.nlp.core.{NCNlpCoreManager, NCNlpPorterStemmer} +import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager import org.apache.nlpcraft.common.pool.NCThreadPoolManager import org.apache.nlpcraft.common.socket.NCSocket import org.apache.nlpcraft.common.version.NCVersion @@ -54,8 +54,7 @@ import scala.util.{Failure, Success} */ object NCProbeManager extends NCService { private final val GSON = new Gson() - private final val TYPE_MODEL_INFO_RESP = new TypeToken[JavaMeta]() {}.getType - private final val TYPE_MODEL_ELEMENT_INFO_RESP = new TypeToken[JavaMeta]() {}.getType + private final val TYPE_JAVA_META = new TypeToken[JavaMeta]() {}.getType // Type safe and eager configuration container. private object Config extends NCConfigurable { @@ -67,7 +66,7 @@ object NCProbeManager extends NCService { def reconnectTimeoutMs: Long = getLong(s"$pre.reconnectTimeoutMs") def pingTimeoutMs: Long = getLong(s"$pre.pingTimeoutMs") def soTimeoutMs: Int = getInt(s"$pre.soTimeoutMs") - + /** * */ @@ -158,8 +157,9 @@ object NCProbeManager extends NCService { // All probes pending complete handshake keyed by probe key. @volatile private var pending: mutable.Map[ProbeKey, ProbeHolder] = _ - @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _ + @volatile private var modelsSynsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _ @volatile private var modelElmsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _ + @volatile private var modelsInfo: ConcurrentHashMap[String, Promise[JavaMeta]] = _ /** * @@ -181,8 +181,9 @@ object NCProbeManager extends NCService { "downlink" -> s"$dnHost:$dnPort" ) - modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]() + modelsSynsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]() modelElmsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]() + modelsInfo = new ConcurrentHashMap[String, Promise[JavaMeta]]() dnSrv = startServer("Downlink", dnHost, dnPort, downLinkHandler) upSrv = startServer("Uplink", upHost, upPort, upLinkHandler) @@ -219,8 +220,9 @@ object NCProbeManager extends NCService { U.stopThread(dnSrv) U.stopThread(upSrv) - modelsInfo = null + modelsSynsInfo = null modelElmsInfo = null + modelsInfo = null ackStopped() } @@ -691,6 +693,23 @@ object NCProbeManager extends NCService { } } } + + /** + * + * @param probeMsg + * @param m + */ + private def processJavaMetaMessage(probeMsg: NCProbeMessage, m: ConcurrentHashMap[String, Promise[JavaMeta]]): Unit = { + val p = m.remove(probeMsg.data[String]("reqGuid")) + + if (p != null) + probeMsg.dataOpt[String]("resp") match { + case Some(resp) => p.success(GSON.fromJson(resp, TYPE_JAVA_META)) + case None => p.failure(new NCE(probeMsg.data[String]("error"))) + } + else + logger.warn(s"Message ignored: $probeMsg") + } /** * Processes the messages received from the probe. @@ -716,27 +735,9 @@ object NCProbeManager extends NCService { typ match { case "P2S_PING" => () - case "P2S_MODEL_INFO" => - val p = modelsInfo.remove(probeMsg.data[String]("reqGuid")) - - if (p != null) - probeMsg.dataOpt[String]("resp") match { - case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_INFO_RESP)) - case None => p.failure(new NCE(probeMsg.data[String]("error"))) - } - else - logger.warn(s"Message ignored: $probeMsg") - - case "P2S_MODEL_ELEMENT_INFO" => - val p = modelElmsInfo.remove(probeMsg.data[String]("reqGuid")) - - if (p != null) - probeMsg.dataOpt[String]("resp") match { - case Some(resp) => p.success(GSON.fromJson(resp, TYPE_MODEL_ELEMENT_INFO_RESP)) - case None => p.failure(new NCE(probeMsg.data[String]("error"))) - } - else - logger.warn(s"Message ignored: $probeMsg") + case "P2S_MODEL_SYNS_INFO" => processJavaMetaMessage(probeMsg, modelsSynsInfo) + case "P2S_MODEL_ELEMENT_INFO" => processJavaMetaMessage(probeMsg, modelElmsInfo) + case "P2S_MODEL_INFO" => processJavaMetaMessage(probeMsg, modelsInfo) case "P2S_ASK_RESULT" => val srvReqId = probeMsg.data[String]("srvReqId") @@ -1089,12 +1090,12 @@ object NCProbeManager extends NCService { * @param parent * @return */ - def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] = - startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ => + def getModelSynonymsInfo(mdlId: String, parent: Span = null): Future[JavaMeta] = + startScopedSpan("getModelSynonymsInfo", parent, "mdlId" -> mdlId) { _ => processModelDataRequest( mdlId, - NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId), - modelsInfo, + NCProbeMessage("S2P_MODEL_SYNS_INFO", "mdlId" -> mdlId), + modelsSynsInfo, parent ) } @@ -1142,4 +1143,15 @@ object NCProbeManager extends NCService { } ) } + + def getModelInfo(mdlId: String, parent: Span = null): Future[JavaMeta] = + startScopedSpan("getModelInfo", parent, "mdlId" -> mdlId) { _ => + processModelDataRequest( + mdlId, + NCProbeMessage("S2P_MODEL_INFO", "mdlId" -> mdlId), + modelsInfo, + parent + ) + } + } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala index e74fb68..39b84e4 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala @@ -879,6 +879,51 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w } } + + /** + * + * @return + */ + protected def $model$info(): Route = { + case class Req$Model$Info( + acsTok: String, + mdlId: String + ) + + implicit val reqFmt: RootJsonFormat[Req$Model$Info] = jsonFormat2(Req$Model$Info) + + entity(as[Req$Model$Info]) { req => + startScopedSpan( + "model$syns", + "acsTok" -> req.acsTok, + "mdlId" -> req.mdlId + ) { span => + checkLength("acsTok" -> req.acsTok, "mdlId" -> req.mdlId) + + val admUsr = authenticateAsAdmin(req.acsTok) + val compId = admUsr.companyId + + if (!NCProbeManager.existsForModel(compId, req.mdlId)) + throw InvalidModelId(req.mdlId) + + val fut = NCProbeManager.getModelInfo(req.mdlId, span) + + successWithJs( + fut.collect { + // We have to use Jackson (not spray) here to serialize 'result' field. + case res => + toJs( + Map( + "status" -> API_OK.toString, + "model" -> res + ) + ) + } + ) + } + } + } + /** * * @return @@ -2040,6 +2085,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w path(API / "probe" / "all") { withMetric(M_PROBE_ALL_LATENCY_MS, $probe$All) } ~ path(API / "model" / "sugsyn") { withMetric(M_MODEL_SUGSYN_LATENCY_MS, $model$sugsyn) } ~ path(API / "model" / "syns") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$syns) } ~ + path(API / "model" / "info") { withMetric(M_MODEL_SYNS_LATENCY_MS, $model$info) } ~ path(API / "ask") { withMetric(M_ASK_LATENCY_MS, $ask) } ~ path(API / "ask" / "sync") { withMetric(M_ASK_SYNC_LATENCY_MS, $ask$Sync) } } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala index fae55d0..d89ba98 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sugsyn/NCSuggestSynonymManager.scala @@ -143,7 +143,7 @@ object NCSuggestSynonymManager extends NCService { val promise = Promise[NCSuggestSynonymResult]() - NCProbeManager.getModelInfo(mdlId, parent).onComplete { + NCProbeManager.getModelSynonymsInfo(mdlId, parent).onComplete { case Success(m) => try { require( diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala index 0b8c03f..d7d22b8 100644 --- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala @@ -111,4 +111,15 @@ class NCRestModelSpec2 extends NCRestSpec { postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elmId" -> ("A" * 65)) postError("model/syns", 400, "NC_ERROR", "mdlId" -> "rest.test.model") } + + @Test + def testModelInfo(): Unit = { + post("model/info", "mdlId" -> "rest.test.model")( + ("$.status", (status: String) => assertEquals("API_OK", status)), + ("$.model", (data: java.util.Map[Object, Object]) => assertTrue(!data.isEmpty)) + ) + + postError("model/info", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN") + postError("model/info", 400, "NC_ERROR") + } }
