This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41 by this push:
new cca3eeb WIP.
cca3eeb is described below
commit cca3eebf1900b23c90950d94002e7663f5ab42cf
Author: Sergey Kamov <[email protected]>
AuthorDate: Sun Aug 30 23:08:20 2020 +0300
WIP.
---
.../nlpcraft/probe/mgrs/cmd/NCCommandManager.scala | 2 +-
.../mgrs/inspections/NCInspectionManager.scala | 6 +-
.../mgrs/inspections/inspectors/NCInspector.scala | 2 +-
.../inspectors/NCInspectorIntents.scala | 2 +-
.../inspections/inspectors/NCInspectorMacros.scala | 2 +-
.../inspectors/NCInspectorSynonyms.scala | 2 +-
.../server/inspections/NCInspectionManager.scala | 18 +-
.../inspections/inspectors/NCInspector.scala | 2 +-
.../inspectors/NCInspectorSuggestions.scala | 511 +++++++++++----------
.../nlpcraft/server/probe/NCProbeManager.scala | 39 +-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 4 +-
11 files changed, 300 insertions(+), 290 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
index 180f531..bc0cfee 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/cmd/NCCommandManager.scala
@@ -96,7 +96,7 @@ object NCCommandManager extends NCService {
NCInspectionManager.inspect(
mdlId = msg.data[String]("mdlId"),
inspId = msg.data[String]("inspId"),
- args = msg.data[String]("args"),
+ args = msg.dataOpt[String]("args"),
span
).collect {
case res ⇒
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
index 2b65771..a2aa3e7 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/NCInspectionManager.scala
@@ -37,7 +37,7 @@ object NCInspectionManager extends NCService with
NCOpenCensusModelStats {
)
override def start(parent: Span): NCService = startScopedSpan("start",
parent) { _ ⇒
- INSPECTORS.values.foreach(_.start())
+ INSPECTORS.values.foreach(_.start(parent))
super.start(parent)
}
@@ -45,7 +45,7 @@ object NCInspectionManager extends NCService with
NCOpenCensusModelStats {
override def stop(parent: Span): Unit = startScopedSpan("stop", parent) {
_ ⇒
super.stop()
- INSPECTORS.values.foreach(_.stop())
+ INSPECTORS.values.foreach(_.stop(parent))
}
/**
@@ -55,7 +55,7 @@ object NCInspectionManager extends NCService with
NCOpenCensusModelStats {
* @param args Inspection arguments.
* @param parent Optional parent trace span.
*/
- def inspect(mdlId: String, inspId: String, args: String, parent: Span =
null): Future[NCInspectionResult] =
+ def inspect(mdlId: String, inspId: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId, "inspectionId" →
inspId) { _ ⇒
INSPECTORS.get(inspId) match {
case Some(inspector) ⇒ inspector.inspect(mdlId, inspId, args)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
index 09adf31..7ada53e 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspector.scala
@@ -23,5 +23,5 @@ import
org.apache.nlpcraft.common.inspections.NCInspectionResult
import scala.concurrent.Future
private[inspections] trait NCInspector {
- def inspect(mdlId: String, inspId: String, args: String, parent: Span =
null): Future[NCInspectionResult]
+ def inspect(mdlId: String, inspId: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult]
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
index 1feab43..31d6352 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorIntents.scala
@@ -34,7 +34,7 @@ import scala.concurrent.Future
object NCInspectorIntents extends NCService with NCInspector {
private final val SEPARATORS = Seq('?', ',', '.', '-', '!')
- override def inspect(mdlId: String, inspId: String, args: String, parent:
Span = null): Future[NCInspectionResult] =
+ override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
Future {
val now = System.currentTimeMillis()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
index eb09059..1f5e8e6 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorMacros.scala
@@ -29,7 +29,7 @@ import scala.concurrent.Future
// TODO:
object NCInspectorMacros extends NCService with NCInspector {
- override def inspect(mdlId: String, inspId: String, args: String, parent:
Span = null): Future[NCInspectionResult] =
+ override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
Future {
val now = System.currentTimeMillis()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
index 112a0dd..78eb001 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/inspections/inspectors/NCInspectorSynonyms.scala
@@ -32,7 +32,7 @@ import scala.concurrent.Future
object NCInspectorSynonyms extends NCService with NCInspector {
private final val TOO_MANY_SYNS = 10000
- override def inspect(mdlId: String, inspId: String, args: String, parent:
Span = null): Future[NCInspectionResult] =
+ override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
Future {
val now = System.currentTimeMillis()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
index 5c46416..ed4340f 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/NCInspectionManager.scala
@@ -42,11 +42,11 @@ object NCInspectionManager extends NCService {
isServerSide = false
),
NCInspectionImpl(
- id = "elements",
- name = "elements",
- synopsis = "elements",
+ id = "intents",
+ name = "intents",
+ synopsis = "intents",
parameters = Seq.empty,
- description = "elements",
+ description = "intents",
isServerSide = false
),
NCInspectionImpl(
@@ -58,11 +58,11 @@ object NCInspectionManager extends NCService {
isServerSide = false
),
NCInspectionImpl(
- id = "synonyms_suggestions",
- name = "synonyms_suggestions",
- synopsis = "synonyms_suggestions",
+ id = "suggestions",
+ name = "suggestions",
+ synopsis = "suggestions",
parameters = Seq.empty,
- description = "synonyms_suggestions",
+ description = "suggestions",
isServerSide = true
)
)
@@ -91,7 +91,7 @@ object NCInspectionManager extends NCService {
* @param args Inspection arguments .
* @param parent Optional parent trace span.
*/
- def inspect(mdlId: String, inspId: String, args: String, parent: Span =
null): Future[NCInspectionResult] =
+ def inspect(mdlId: String, inspId: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult] =
SRV_INSPECTORS.get(inspId) match {
case Some(inspector) ⇒ inspector.inspect(mdlId, inspId, args,
parent)
case None ⇒ NCProbeManager.getProbeInspection(mdlId, inspId, args,
parent)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
index 9e985d4..d455ab5 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspector.scala
@@ -6,5 +6,5 @@ import org.apache.nlpcraft.common.inspections.NCInspectionResult
import scala.concurrent.Future
private[inspections] trait NCInspector {
- def inspect(mdlId: String, inspId: String, args: String, parent: Span =
null): Future[NCInspectionResult]
+ def inspect(mdlId: String, inspId: String, args: Option[String], parent:
Span = null): Future[NCInspectionResult]
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
index f9a0259..28f9580 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/inspections/inspectors/NCInspectorSuggestions.scala
@@ -82,6 +82,7 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
)
}
+ // TODO:
case class SuggestionResult(
synonym: String,
ctxWorldServerScore: Double,
@@ -136,7 +137,7 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
seq
}
- override def inspect(mdlId: String, inspId: String, args: String, parent:
Span = null): Future[NCInspectionResult] =
+ override def inspect(mdlId: String, inspId: String, args: Option[String],
parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId) { _ ⇒
val now = System.currentTimeMillis()
@@ -144,298 +145,308 @@ object NCInspectorSuggestions extends NCService with
NCInspector {
NCProbeManager.getModelInfo(mdlId, parent).collect {
case data ⇒
- println("args=" + args)
println("data=" + data)
+ println("args=" + args)
- val macrosJ =
data.get("macros").asInstanceOf[util.Map[String, String]]
- val elementsSynonymsJ =
data.get("elementsSynonyms").asInstanceOf[util.Map[String, util.List[String]]]
- val intentsSamplesJ =
data.get("intentsSamples").asInstanceOf[util.Map[String, util.List[String]]]
-
- require(macrosJ != null)
- require(elementsSynonymsJ != null)
- require(intentsSamplesJ != null)
-
- val macros = macrosJ.asScala
- val elementsSynonyms = elementsSynonymsJ.asScala.map(p ⇒
p._1 → p._2.asScala)
- val intentsSamples = intentsSamplesJ.asScala.map(p ⇒ p._1
→ p._2.asScala)
-
- def onError(err: String): Unit =
- promise.success(
- NCInspectionResultImpl(
- inspectionId = inspId,
- modelId = mdlId,
- inspectionArguments = None,
- durationMs = System.currentTimeMillis() - now,
- timestamp = now,
- errors = Seq(err),
- warnings = Seq.empty,
- suggestions = Seq.empty
- )
- )
-
- if (intentsSamples.isEmpty)
- onError(s"Missed intents samples for: '$mdlId'")
- else {
- val url = s"${Config.urlOpt.getOrElse(throw new
NCE("Context word server is not configured"))}/suggestions"
+ try {
+ val macrosJ =
data.get("macros").asInstanceOf[util.Map[String, String]]
+ val elementsSynonymsJ =
data.get("elementsSynonyms").asInstanceOf[util.Map[String, util.List[String]]]
+ val intentsSamplesJ =
data.get("intentsSamples").asInstanceOf[util.Map[String, util.List[String]]]
- val allSamplesCnt = intentsSamples.map { case (_,
samples) ⇒ samples.size }.sum
+ require(macrosJ != null)
+ require(elementsSynonymsJ != null)
+ require(intentsSamplesJ != null)
- val warns = mutable.ArrayBuffer.empty[String]
+ val macros = macrosJ.asScala
+ val elementsSynonyms = elementsSynonymsJ.asScala.map(p
⇒ p._1 → p._2.asScala)
+ val intentsSamples = intentsSamplesJ.asScala.map(p ⇒
p._1 → p._2.asScala)
- if (allSamplesCnt < MIN_CNT_MODEL)
- warns +=
- s"Model: '$mdlId' has too small intents
samples count: $allSamplesCnt. " +
- s"Potentially is can be not enough for
suggestions service high quality work. " +
- s"Try to increase their count at least to
$MIN_CNT_MODEL."
+ def onError(err: String): Unit =
+ promise.success(
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs = System.currentTimeMillis() -
now,
+ timestamp = now,
+ errors = Seq(err),
+ warnings = Seq.empty,
+ suggestions = Seq.empty
+ )
+ )
+ if (intentsSamples.isEmpty)
+ onError(s"Missed intents samples for: '$mdlId'")
else {
- val ids =
- intentsSamples.
- filter { case (_, samples) ⇒ samples.size
< MIN_CNT_INTENT }.
- map { case (intentId, _) ⇒ intentId }
-
- if (ids.nonEmpty)
- warns +=
- s"Models '$mdlId' has intents:
[${ids.mkString(", ")}] with too small intents samples count." +
- s"Potentially it can be not enough for
suggestions service high quality work. " +
- s"Try to increase their count at least
to $MIN_CNT_INTENT."
- }
-
- val parser = new NCMacroParser()
-
- macros.foreach { case (name, str) ⇒
parser.addMacro(name, str) }
-
- // Note that we don't use system tokenizer, because
ContextWordServer doesn't have this tokenizer.
- // We just split examples words with spaces. Also we
divide SEPARATORS as separated words.
- val examples =
- intentsSamples.
- flatMap { case (_, samples) ⇒ samples }.
- map(ex ⇒ SEPARATORS.foldLeft(ex)((s, ch) ⇒
s.replaceAll(s"\\$ch", s" $ch "))).
- map(ex ⇒ {
- val seq = ex.split(" ")
-
- seq → seq.map(toStemWord)
- }).
- toMap
-
- val elemSyns =
- elementsSynonyms.map { case (elemId, syns) ⇒
elemId → syns.flatMap(parser.expand) }.
- map { case (id, seq) ⇒ id → seq.map(txt ⇒
split(txt).map(p ⇒ Word(p, toStemWord(p)))) }
-
- val allReqs =
- elemSyns.map {
- case (elemId, syns) ⇒
- val normSyns: Seq[Seq[Word]] =
syns.filter(_.size == 1)
- val synsStems = normSyns.map(_.map(_.stem))
- val synsWords = normSyns.map(_.map(_.word))
-
- val reqs =
- examples.flatMap { case (exampleWords,
exampleStems) ⇒
- val exampleIdxs =
synsStems.flatMap(synStems ⇒ getAllSlices(exampleStems, synStems))
-
- def mkRequestData(idx: Int,
synStems: Seq[String], synStemsIdx: Int): RequestData = {
- val fromIncl = idx
- val toExcl = idx +
synStems.length
-
- RequestData(
- sentence =
exampleWords.zipWithIndex.flatMap {
- case (exampleWord, i) ⇒
- i match {
- case x if x ==
fromIncl ⇒ synsWords(synStemsIdx)
- case x if x >
fromIncl && x < toExcl ⇒ Seq.empty
- case _ ⇒
Seq(exampleWord)
- }
- }.mkString(" "),
- example =
exampleWords.mkString(" "),
- elementId = elemId,
- index = idx
- )
- }
+ val url = s"${Config.urlOpt.getOrElse(throw new
NCE("Context word server is not configured"))}/suggestions"
- (for (idx ← exampleIdxs;
(synStems, i) ← synsStems.zipWithIndex)
- yield mkRequestData(idx,
synStems, i)).distinct
- }
+ val allSamplesCnt = intentsSamples.map { case (_,
samples) ⇒ samples.size }.sum
- elemId → reqs.toSet
- }.filter(_._2.nonEmpty)
+ val warns = mutable.ArrayBuffer.empty[String]
- val noExElems =
- elementsSynonyms.
- filter { case (elemId, syns) ⇒ syns.nonEmpty
&& !allReqs.contains(elemId) }.
- map { case (elemId, _) ⇒ elemId }
-
- if (noExElems.nonEmpty)
- warns +=
- "Some elements don't have synonyms in their
intent samples, " +
- s"so the service can't suggest any new
synonyms for such elements: [${noExElems.mkString(", ")}]"
+ if (allSamplesCnt < MIN_CNT_MODEL)
+ warns +=
+ s"Model: '$mdlId' has too small intents
samples count: $allSamplesCnt. " +
+ s"Potentially is can be not enough for
suggestions service high quality work. " +
+ s"Try to increase their count at least
to $MIN_CNT_MODEL."
+
+ else {
+ val ids =
+ intentsSamples.
+ filter { case (_, samples) ⇒
samples.size < MIN_CNT_INTENT }.
+ map { case (intentId, _) ⇒ intentId }
+
+ if (ids.nonEmpty)
+ warns +=
+ s"Models '$mdlId' has intents:
[${ids.mkString(", ")}] with too small intents samples count." +
+ s"Potentially it can be not enough
for suggestions service high quality work. " +
+ s"Try to increase their count at
least to $MIN_CNT_INTENT."
+ }
- val allReqsCnt = allReqs.map(_._2.size).sum
- val allSynsCnt = elemSyns.map(_._2.size).sum
+ val parser = new NCMacroParser()
+
+ macros.foreach { case (name, str) ⇒
parser.addMacro(name, str) }
+
+ // Note that we don't use system tokenizer,
because ContextWordServer doesn't have this tokenizer.
+ // We just split examples words with spaces. Also
we divide SEPARATORS as separated words.
+ val examples =
+ intentsSamples.
+ flatMap { case (_, samples) ⇒ samples }.
+ map(ex ⇒ SEPARATORS.foldLeft(ex)((s, ch) ⇒
s.replaceAll(s"\\$ch", s" $ch "))).
+ map(ex ⇒ {
+ val seq = ex.split(" ")
+
+ seq → seq.map(toStemWord)
+ }).
+ toMap
+
+ val elemSyns =
+ elementsSynonyms.map { case (elemId, syns) ⇒
elemId → syns.flatMap(parser.expand) }.
+ map { case (id, seq) ⇒ id → seq.map(txt ⇒
split(txt).map(p ⇒ Word(p, toStemWord(p)))) }
+
+ val allReqs =
+ elemSyns.map {
+ case (elemId, syns) ⇒
+ val normSyns: Seq[Seq[Word]] =
syns.filter(_.size == 1)
+ val synsStems =
normSyns.map(_.map(_.stem))
+ val synsWords =
normSyns.map(_.map(_.word))
+
+ val reqs =
+ examples.flatMap { case
(exampleWords, exampleStems) ⇒
+ val exampleIdxs =
synsStems.flatMap(synStems ⇒ getAllSlices(exampleStems, synStems))
+
+ def mkRequestData(idx: Int,
synStems: Seq[String], synStemsIdx: Int): RequestData = {
+ val fromIncl = idx
+ val toExcl = idx +
synStems.length
+
+ RequestData(
+ sentence =
exampleWords.zipWithIndex.flatMap {
+ case (exampleWord,
i) ⇒
+ i match {
+ case x if
x == fromIncl ⇒ synsWords(synStemsIdx)
+ case x if
x > fromIncl && x < toExcl ⇒ Seq.empty
+ case _ ⇒
Seq(exampleWord)
+ }
+ }.mkString(" "),
+ example =
exampleWords.mkString(" "),
+ elementId = elemId,
+ index = idx
+ )
+ }
- logger.info(s"Data prepared. Request is going to
execute on ContextWord Server " +
- s"[examples=${examples.size}, " +
- s"synonyms=$allSynsCnt, " +
- s"requests=$allReqsCnt]"
- )
+ (for (idx ← exampleIdxs;
(synStems, i) ← synsStems.zipWithIndex)
+ yield mkRequestData(idx,
synStems, i)).distinct
+ }
- if (allReqsCnt == 0)
- onError(s"Suggestions cannot be prepared:
'$mdlId'. Samples don't contain synonyms")
- else {
- val allSuggs = new ConcurrentHashMap[String,
JList[Suggestion]]()
- val cdl = new CountDownLatch(1)
- val debugs = mutable.HashMap.empty[RequestData,
Seq[Suggestion]]
- val cnt = new AtomicInteger(0)
-
- val client = HttpClients.createDefault
- val err = new AtomicReference[Throwable]()
-
- for ((elemId, reqs) ← allReqs; batch ←
reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
- NCUtils.asFuture(
- _ ⇒ {
- val post = new HttpPost(url)
-
- post.setHeader("Content-Type",
"application/json")
-
- post.setEntity(
- new StringEntity(
- GSON.toJson(
- RestRequest(
- sentences =
batch.map(p ⇒ RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
- // ContextWord server
range is (0, 2), input range is (0, 1)
- min_score =
Config.suggestionsMinScore * 2,
- // We set big limit
value and in fact only minimal score is taken into account.
- limit = MAX_LIMIT
- )
- ),
- "UTF-8"
- )
- )
+ elemId → reqs.toSet
+ }.filter(_._2.nonEmpty)
- val resps: Seq[Seq[Suggestion]] =
- try
- client.execute(post, HANDLER)
- finally
- post.releaseConnection()
+ val noExElems =
+ elementsSynonyms.
+ filter { case (elemId, syns) ⇒
syns.nonEmpty && !allReqs.contains(elemId) }.
+ map { case (elemId, _) ⇒ elemId }
- require(batch.size == resps.size,
s"Batch: ${batch.size}, responses: ${resps.size}")
+ if (noExElems.nonEmpty)
+ warns +=
+ "Some elements don't have synonyms in
their intent samples, " +
+ s"so the service can't suggest any new
synonyms for such elements: [${noExElems.mkString(", ")}]"
- batch.zip(resps).foreach { case (req,
resp) ⇒ debugs += req → resp }
+ val allReqsCnt = allReqs.map(_._2.size).sum
+ val allSynsCnt = elemSyns.map(_._2.size).sum
- val i = cnt.addAndGet(batch.size)
+ logger.info(s"Data prepared. Request is going to
execute on ContextWord Server " +
+ s"[examples=${examples.size}, " +
+ s"synonyms=$allSynsCnt, " +
+ s"requests=$allReqsCnt]"
+ )
- logger.debug(s"Executed: $i
requests...")
+ if (allReqsCnt == 0)
+ onError(s"Suggestions cannot be prepared:
'$mdlId'. Samples don't contain synonyms")
+ else {
+ val allSuggs = new ConcurrentHashMap[String,
JList[Suggestion]]()
+ val cdl = new CountDownLatch(1)
+ val debugs =
mutable.HashMap.empty[RequestData, Seq[Suggestion]]
+ val cnt = new AtomicInteger(0)
+
+ val client = HttpClients.createDefault
+ val err = new AtomicReference[Throwable]()
+
+ for ((elemId, reqs) ← allReqs; batch ←
reqs.sliding(BATCH_SIZE, BATCH_SIZE).map(_.toSeq)) {
+ NCUtils.asFuture(
+ _ ⇒ {
+ val post = new HttpPost(url)
+
+ post.setHeader("Content-Type",
"application/json")
+
+ post.setEntity(
+ new StringEntity(
+ GSON.toJson(
+ RestRequest(
+ sentences =
batch.map(p ⇒ RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
+ // ContextWord
server range is (0, 2), input range is (0, 1)
+ min_score =
Config.suggestionsMinScore * 2,
+ // We set big
limit value and in fact only minimal score is taken into account.
+ limit = MAX_LIMIT
+ )
+ ),
+ "UTF-8"
+ )
+ )
- allSuggs.
- computeIfAbsent(elemId, (_:
String) ⇒ new CopyOnWriteArrayList[Suggestion]()).
- addAll(resps.flatten.asJava)
+ val resps: Seq[Seq[Suggestion]] =
+ try
+ client.execute(post,
HANDLER)
+ finally
+ post.releaseConnection()
- if (i == allReqsCnt)
- cdl.countDown()
- },
- (e: Throwable) ⇒ {
- err.compareAndSet(null, e)
+ require(batch.size == resps.size,
s"Batch: ${batch.size}, responses: ${resps.size}")
- cdl.countDown()
- },
- (_: Unit) ⇒ ()
- )
- }
+ batch.zip(resps).foreach { case
(req, resp) ⇒ debugs += req → resp }
- cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
+ val i = cnt.addAndGet(batch.size)
- if (err.get() != null)
- throw new NCE("Error during work with
ContextWord Server", err.get())
+ logger.debug(s"Executed: $i
requests...")
- val allSynsStems =
elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
+ allSuggs.
+ computeIfAbsent(elemId, (_:
String) ⇒ new CopyOnWriteArrayList[Suggestion]()).
+ addAll(resps.flatten.asJava)
- val nonEmptySuggs = allSuggs.asScala.map(p ⇒ p._1
→ p._2.asScala).filter(_._2.nonEmpty)
+ if (i == allReqsCnt)
+ cdl.countDown()
+ },
+ (e: Throwable) ⇒ {
+ err.compareAndSet(null, e)
- val res = mutable.HashMap.empty[String,
mutable.ArrayBuffer[SuggestionResult]]
+ cdl.countDown()
+ },
+ (_: Unit) ⇒ ()
+ )
+ }
- nonEmptySuggs.
- foreach { case (elemId, elemSuggs) ⇒
- elemSuggs.
- map(sugg ⇒ (sugg, toStem(sugg.word))).
- groupBy { case (_, stem) ⇒ stem }.
- // Drops already defined.
- filter { case (stem, _) ⇒
!allSynsStems.contains(stem) }.
- map { case (_, group) ⇒
- val seq = group.map { case (sugg,
_) ⇒ sugg }.sortBy(-_.score)
+ cdl.await(Long.MaxValue, TimeUnit.MILLISECONDS)
- // Drops repeated.
- (seq.head, seq.length)
- }.
- toSeq.
- map { case (sugg, cnt) ⇒ (sugg, cnt,
sugg.score * cnt / elemSuggs.size) }.
- sortBy { case (_, _, sumFactor) ⇒
-sumFactor }.
- zipWithIndex.
- foreach { case ((sugg, cnt, _), _) ⇒
- val seq =
- res.get(elemId) match {
- case Some(seq) ⇒ seq
- case None ⇒
- val buf =
mutable.ArrayBuffer.empty[SuggestionResult]
+ if (err.get() != null)
+ throw new NCE("Error during work with
ContextWordServer", err.get())
- res += elemId → buf
+ val allSynsStems =
elemSyns.flatMap(_._2).flatten.map(_.stem).toSet
- buf
- }
+ val nonEmptySuggs = allSuggs.asScala.map(p ⇒
p._1 → p._2.asScala).filter(_._2.nonEmpty)
- seq += SuggestionResult(sugg.word,
sugg.score, cnt)
- }
- }
+ val res = mutable.HashMap.empty[String,
mutable.ArrayBuffer[SuggestionResult]]
- logger.whenDebugEnabled({
- logger.debug("Request information:")
+ nonEmptySuggs.
+ foreach { case (elemId, elemSuggs) ⇒
+ elemSuggs.
+ map(sugg ⇒ (sugg,
toStem(sugg.word))).
+ groupBy { case (_, stem) ⇒ stem }.
+ // Drops already defined.
+ filter { case (stem, _) ⇒
!allSynsStems.contains(stem) }.
+ map { case (_, group) ⇒
+ val seq = group.map { case
(sugg, _) ⇒ sugg }.sortBy(-_.score)
- var i = 1
+ // Drops repeated.
+ (seq.head, seq.length)
+ }.
+ toSeq.
+ map { case (sugg, cnt) ⇒ (sugg,
cnt, sugg.score * cnt / elemSuggs.size) }.
+ sortBy { case (_, _, sumFactor) ⇒
-sumFactor }.
+ zipWithIndex.
+ foreach { case ((sugg, cnt, _), _)
⇒
+ val seq =
+ res.get(elemId) match {
+ case Some(seq) ⇒ seq
+ case None ⇒
+ val buf =
mutable.ArrayBuffer.empty[SuggestionResult]
- debugs.groupBy(_._1.example).foreach { case
(_, m) ⇒
- m.toSeq.sortBy(_._1.sentence).foreach {
case (req, suggs) ⇒
- val s =
- split(req.sentence).
- zipWithIndex.map { case (w, i)
⇒ if (i == req.index) s"<<<$w>>>" else w }.
- mkString(" ")
+ res += elemId → buf
- logger.debug(
- s"$i. " +
- s"Request=$s, " +
-
s"suggestions=[${suggs.map(_.word).mkString(", ")}], " +
- s"element=${req.elementId}"
- )
+ buf
+ }
- i = i + 1
+ seq +=
SuggestionResult(sugg.word, sugg.score, cnt)
+ }
}
- }
- })
- val resJ: util.Map[String,
JList[util.HashMap[String, Any]]] =
- res.map { case (id, data) ⇒
- id → data.map(d ⇒ {
- val m = new util.HashMap[String, Any]()
+ logger.whenDebugEnabled({
+ logger.debug("Request information:")
- m.put("synonym", d.synonym)
- m.put("ctxWorldServerScore",
d.ctxWorldServerScore)
- m.put("suggestedCount",
d.suggestedCount)
+ var i = 1
- m
- }).asJava
- }.asJava
+ debugs.groupBy(_._1.example).foreach {
case (_, m) ⇒
+ m.toSeq.sortBy(_._1.sentence).foreach
{ case (req, suggs) ⇒
+ val s =
+ split(req.sentence).
+ zipWithIndex.map { case
(w, i) ⇒ if (i == req.index) s"<<<$w>>>" else w }.
+ mkString(" ")
- promise.success(
- NCInspectionResultImpl(
- inspectionId = inspId,
- modelId = mdlId,
- inspectionArguments = None,
- durationMs = System.currentTimeMillis() -
now,
- timestamp = now,
- errors = Seq.empty,
- warnings = warns,
- suggestions = Seq(resJ)
+ logger.debug(
+ s"$i. " +
+ s"Request=$s, " +
+
s"suggestions=[${suggs.map(_.word).mkString(", ")}], " +
+ s"element=${req.elementId}"
+ )
+
+ i = i + 1
+ }
+ }
+ })
+
+ val resJ: util.Map[String,
JList[util.HashMap[String, Any]]] =
+ res.map { case (id, data) ⇒
+ id → data.map(d ⇒ {
+ val m = new util.HashMap[String,
Any]()
+
+ m.put("synonym", d.synonym)
+ m.put("ctxWorldServerScore",
d.ctxWorldServerScore)
+ m.put("suggestedCount",
d.suggestedCount)
+
+ m
+ }).asJava
+ }.asJava
+
+ promise.success(
+ NCInspectionResultImpl(
+ inspectionId = inspId,
+ modelId = mdlId,
+ inspectionArguments = None,
+ durationMs =
System.currentTimeMillis() - now,
+ timestamp = now,
+ errors = Seq.empty,
+ warnings = warns,
+ suggestions = Seq(resJ)
+ )
)
- )
- }
+ }
}
+ }
+ catch {
+ case e: NCE ⇒ promise.failure(e)
+ case e: Throwable ⇒
+ logger.warn("Unexpected error.", e)
+
+ promise.failure(e)
+ }
+
}(scala.concurrent.ExecutionContext.Implicits.global)
promise.future
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
index 3a957d7..0616a4a 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/probe/NCProbeManager.scala
@@ -148,7 +148,7 @@ object NCProbeManager extends NCService {
@volatile private var isStopping: AtomicBoolean = _
@volatile private var probeInspecs: ConcurrentHashMap[String,
Promise[NCInspectionResult]] = _
- @volatile private var modelsInfo: ConcurrentHashMap[String,
Promise[Map[String, AnyRef]]] = _
+ @volatile private var modelsInfo: ConcurrentHashMap[String,
Promise[java.util.Map[String, AnyRef]]] = _
/**
*
@@ -170,7 +170,7 @@ object NCProbeManager extends NCService {
isStopping = new AtomicBoolean(false)
probeInspecs = new ConcurrentHashMap[String,
Promise[NCInspectionResult]]()
- modelsInfo = new ConcurrentHashMap[String, Promise[Map[String,
AnyRef]]]()
+ modelsInfo = new ConcurrentHashMap[String,
Promise[java.util.Map[String, AnyRef]]]()
pool = Executors.newFixedThreadPool(Config.poolSize)
@@ -693,11 +693,7 @@ object NCProbeManager extends NCService {
if (promise != null) {
val r: T = GSON.fromJson(probeMsg.data[String]("resp"),
typ)
- println("+!r=" + r)
-
promise.success(r)
-
- println("+!r!!!=" + r)
}
}
@@ -987,17 +983,20 @@ object NCProbeManager extends NCService {
* @param parent
* @return
*/
- def getProbeInspection(mdlId: String, inspId: String, args: String,
parent: Span = null): Future[NCInspectionResult] =
+ def getProbeInspection(mdlId: String, inspId: String, args:
Option[String], parent: Span = null): Future[NCInspectionResult] =
startScopedSpan("inspect", parent, "modelId" → mdlId, "inspId" →
inspId) { _ ⇒
- probePromise(
- parent,
- mdlId,
- probeInspecs,
- "S2P_PROBE_INSPECTION",
- "mdlId" → mdlId,
- "inspId" → inspId,
- "args" → GSON.toJson(args)
- )
+ val m =
+ Map(
+ "mdlId" → mdlId,
+ "inspId" → inspId,
+ "args" →
+ (args match {
+ case Some(a) ⇒ GSON.toJson(a)
+ case None ⇒ null
+ })
+ )
+
+ probePromise(parent, mdlId, probeInspecs, "S2P_PROBE_INSPECTION",
m.filter(_._2 != null).toSeq :_*)
}
/**
@@ -1006,7 +1005,7 @@ object NCProbeManager extends NCService {
* @param parent
* @return
*/
- def getModelInfo(mdlId: String, parent: Span = null): Future[Map[String,
AnyRef]] =
+ def getModelInfo(mdlId: String, parent: Span = null):
Future[java.util.Map[String, AnyRef]] =
startScopedSpan("getModelInfo", parent, "modelId" → mdlId) { _ ⇒
probePromise(
parent,
@@ -1023,18 +1022,18 @@ object NCProbeManager extends NCService {
* @param mdlId
* @param promises
* @param msgId
- * @param mdgParams
+ * @param msgParams
*/
private def probePromise[T](
parent: Span,
mdlId: String,
promises: ConcurrentHashMap[String, Promise[T]],
msgId: String,
- mdgParams: (String, Serializable)*
+ msgParams: (String, Serializable)*
): Future[T] =
getProbeForModelId(mdlId) match {
case Some(probe) ⇒
- val msg = NCProbeMessage(msgId, mdgParams:_*)
+ val msg = NCProbeMessage(msgId, msgParams:_*)
val promise = Promise[T]()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 88be3f9..b293b56 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -661,7 +661,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
if (!NCProbeManager.getAllProbes(admin.companyId,
span).exists(_.models.exists(_.id == mdlId)))
throw new NCE(s"Probe not found for model: $mdlId")
- NCInspectionManager.inspect(mdlId, inspId, args.orNull,
span).collect {
+ NCInspectionManager.inspect(mdlId, inspId, args, span).collect {
// We have to use GSON (not spray) here to serialize `result`
field.
case res ⇒ GSON.toJson(Map("status" → API_OK.toString,
"result" → res).asJava)
}
@@ -1692,7 +1692,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
val code = "NC_ERROR"
// We have to log error reason because even general exceptions are
not expected here.
- logger.warn(s"Unexpected error: $errMsg", e)
+ logger.warn(s"Unexpected top level REST API error: $errMsg", e)
completeError(StatusCodes.BadRequest, code, errMsg)
