This is an automated email from the ASF dual-hosted git repository. sergeykamov pushed a commit to branch NLPCRAFT-415 in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit ababa685c11ccf31e5dc8e7d6746be502c9c8c66 Author: Sergey Kamov <[email protected]> AuthorDate: Sun Aug 22 21:11:15 2021 +0300 WIP. --- .../nlpcraft/server/probe/NCProbeManager.scala | 45 ++++++++++++++++++---- .../nlpcraft/server/rest/NCBasicRestApi.scala | 22 +++++++++-- .../nlpcraft/server/rest/NCRestModelSpec.scala | 39 +++++++++++++------ 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala index cd0d5a5..ad9f84a 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala @@ -44,6 +44,7 @@ import java.security.Key import java.util import java.util.Collections import java.util.concurrent.ConcurrentHashMap +import java.util.regex.Pattern import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future, Promise} import scala.jdk.CollectionConverters.{ListHasAsScala, MapHasAsJava, MapHasAsScala, SeqHasAsJava, SetHasAsScala} @@ -1105,10 +1106,11 @@ object NCProbeManager extends NCService { * * @param mdlId * @param elmId + * @param pattern * @param parent * @return */ - def getModelElementInfo(mdlId: String, elmId: String, parent: Span = null): Future[JavaMeta] = + def getModelElementInfo(mdlId: String, elmId: String, pattern: Option[String], parent: Span = null): Future[JavaMeta] = startScopedSpan("getModelElementInfo", parent, "mdlId" -> mdlId, "elmId" -> elmId) { _ => processModelInfoRequest( mdlId, @@ -1124,16 +1126,44 @@ object NCProbeManager extends NCService { ) val macros = res.remove("macros").asInstanceOf[java.util.Map[String, String]].asScala - val syns = res.get("synonyms").asInstanceOf[java.util.List[String]].asScala - val vals = res.get("values").asInstanceOf[java.util.Map[String, java.util.List[String]]].asScala + + var syns = res.remove("synonyms").asInstanceOf[java.util.List[String]].asScala + var vals = res.remove("values").asInstanceOf[java.util.Map[String, java.util.List[String]]].asScala val parser = NCMacroParser(macros.toList) - val synsExp = syns.flatMap(s => parser.expand(s)).sorted - val valsExp = vals.map(v => v._1 -> v._2.asScala.flatMap(s => parser.expand(s)).sorted.asJava).toMap + var synsExp: Seq[(String, Seq[String])] = + syns.map(s => s -> parser.expand(s)) + var valsExp: Map[String, mutable.Buffer[(String, Seq[String])]] = + vals.map(v => v._1 -> v._2.asScala.map(s => s -> parser.expand(s))).toMap + + pattern match { + case Some(p) => + val regex = Pattern.compile(p) + + def any(s: String): Boolean = regex.matcher(s).find() - res.put("synonymsExp", synsExp.asJava) - res.put("valuesExp", valsExp.asJava) + synsExp = synsExp.map(p => p._1 -> p._2.filter(any)).filter(_._2.nonEmpty) + syns = syns.filter(s => synsExp.exists(_._1 == s)) + + valsExp = valsExp.flatMap { case (name, syns) => + val synsFiltered = syns.map(p => p._1 -> p._2.filter(any)).filter(_._2.nonEmpty) + + if (any(name) || synsFiltered.nonEmpty) + Some(name -> synsFiltered) + else + None + } + vals = vals.map { case (name, syns) => name -> syns.asScala.filter(s => + valsExp.exists { case (valExpName, valExpSyns) => name == valExpName && valExpSyns.exists(_._1 == s)} + )}.filter(_._2.nonEmpty).map(p => p._1 -> p._2.asJava) + case None => // No-op. + } + + res.put("synonyms", syns.asJava) + res.put("values", vals.asJava) + res.put("synonymsExp", synsExp.flatMap(_._2).sorted.asJava) + res.put("valuesExp", valsExp.map(p => p._1 -> p._2.flatMap(_._2).sorted).asJava) // Add statistics. res.put("synonymsExpCnt", Integer.valueOf(synsExp.size)) @@ -1154,5 +1184,4 @@ object NCProbeManager extends NCService { parent ) } - } diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala index 2bbe409..db4c6bc 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala @@ -48,6 +48,7 @@ import org.apache.nlpcraft.server.user.NCUserManager import spray.json.DefaultJsonProtocol._ import spray.json.{JsObject, JsValue, RootJsonFormat} +import java.util.regex.{Pattern, PatternSyntaxException} import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ @@ -134,7 +135,9 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w "comment" -> 1024, "avatarUrl" -> 512000, - "adminAvatarUrl" -> 512000 + "adminAvatarUrl" -> 512000, + + "pattern" -> 512 ) /** @@ -830,10 +833,11 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w case class Req$Model$Syns( acsTok: String, mdlId: String, - elmId: String + elmId: String, + pattern: Option[String] ) - implicit val reqFmt: RootJsonFormat[Req$Model$Syns] = jsonFormat3(Req$Model$Syns) + implicit val reqFmt: RootJsonFormat[Req$Model$Syns] = jsonFormat4(Req$Model$Syns) entity(as[Req$Model$Syns]) { req => startScopedSpan( @@ -851,7 +855,17 @@ class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace w if (!NCProbeManager.existsForModelElement(compId, req.mdlId, req.elmId)) throw InvalidModelOrElementId(req.mdlId, req.elmId) - val fut = NCProbeManager.getModelElementInfo(req.mdlId, req.elmId, span) + req.pattern match { + case Some(pattern) => + try + Pattern.compile(pattern) + catch { + case _ : PatternSyntaxException => throw InvalidField(pattern) + } + case None => // No-op. + } + + val fut = NCProbeManager.getModelElementInfo(req.mdlId, req.elmId, req.pattern, span) successWithJs( fut.collect { diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala index f68544d..9cc5edc 100644 --- a/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/server/rest/NCRestModelSpec.scala @@ -76,7 +76,13 @@ class RestTestModelExt extends RestTestModel { super.getElements.asScala ++ Set( NCTestElement("eExt1", "<M1>", "<M1> more"), - NCTestElement("eExt2", Seq("<M1>", "<M1> more"), Map("v1"-> Seq("<M2>", "<M2> more"), "v2" -> Seq("<M2>"))) + NCTestElement("eExt2", + Seq("<M1>", "<M1> more <M2>"), + Map( + "v1"-> Seq("<M2>", "<M2> more"), + "v2" -> Seq("<M2>"), + "v3" -> Seq("<M1>")) + ) ) ).asJava } @@ -89,20 +95,29 @@ class NCRestModelSpec2 extends NCRestSpec { @Test def testSyns(): Unit = { // Note that checked values are valid for current configuration of `RestTestModelExt` model. - def post0(elemId: String, valsShouldBe: Boolean): Unit = - post("model/syns", "mdlId" -> "rest.test.model", "elmId" -> elemId)( + def post0( + elemId: String, + pattern: Option[String] = None, + checkSyns: Integer => Boolean = _ > 0, + checkSynsExp: Integer => Boolean = _ > 0, + checkVals: Integer => Boolean = _ > 0, + checkValsExp: Integer => Boolean = _ > 0 + ): Unit = + post("model/syns", "mdlId" -> "rest.test.model", "elmId" -> elemId, "pattern" -> pattern.orNull)( ("$.status", (status: String) => assertEquals("API_OK", status)), - ("$.synonyms", (data: ResponseList) => assertTrue(!data.isEmpty)), - ("$.synonymsExp", (data: ResponseList) => assertTrue(!data.isEmpty)), - ("$.values", (data: java.util.Map[Object, Object]) => - if (valsShouldBe) assertTrue(!data.isEmpty) else assertTrue(data.isEmpty)), - ("$.valuesExp", (data: java.util.Map[Object, Object]) => - if (valsShouldBe) assertTrue(!data.isEmpty) else assertTrue(data.isEmpty) - ) + ("$.synonyms", (data: ResponseList) => assertTrue(checkSyns(data.size()))), + ("$.synonymsExp", (data: ResponseList) => assertTrue(checkSynsExp(data.size()))), + ("$.values", (data: java.util.Map[Object, Object]) => assertTrue(checkVals(data.size()))), + ("$.valuesExp", (data: java.util.Map[Object, Object]) => assertTrue(checkValsExp(data.size()))) ) - post0("eExt1", valsShouldBe = false) - post0("eExt2", valsShouldBe = true) + + post0("eExt1", checkVals = _ == 0, checkValsExp = _ == 0) + post0("eExt2", checkSyns = _ == 3, checkSynsExp = _ == 11, checkVals = _ == 3, checkValsExp = _ == 3) + + post0("eExt2", pattern = Some("mtest2"), checkSyns = _ == 1, checkSynsExp = _ == 8, checkVals = _ == 2, checkValsExp = _ == 2) + post0("eExt2", pattern = Some("UNKNOWN"), checkSyns = _ == 0, checkSynsExp = _ == 0, checkVals = _ == 0, checkValsExp = _ == 0) + postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "UNKNOWN", "elmId" -> "UNKNOWN") postError("model/syns", 400, "NC_INVALID_FIELD", "mdlId" -> "rest.test.model", "elmId" -> "UNKNOWN")
