This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-41
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-41 by this push:
new 082e434 WIP.
082e434 is described below
commit 082e43420f381e60404a31b2da44412b34aa9fe8
Author: Sergey Kamov <[email protected]>
AuthorDate: Wed Aug 19 16:36:09 2020 +0300
WIP.
---
.../org/apache/nlpcraft/server/NCServer.scala | 6 +-
.../NCEnhanceManager.scala} | 24 ++++--
.../NCEnhanceSuggestion.scala} | 8 +-
.../opencensus/NCOpenCensusServerStats.scala | 2 +-
.../nlpcraft/server/rest/NCBasicRestApi.scala | 95 +++++++++++-----------
openapi/nlpcraft_swagger.yml | 10 +--
6 files changed, 76 insertions(+), 69 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
index 7ac8179..34e85d0 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
@@ -45,7 +45,7 @@ import org.apache.nlpcraft.server.proclog.NCProcessLogManager
import org.apache.nlpcraft.server.query.NCQueryManager
import org.apache.nlpcraft.server.rest.NCRestManager
import org.apache.nlpcraft.server.sql.NCSqlManager
-import org.apache.nlpcraft.server.suggestion.NCSuggestionsManager
+import org.apache.nlpcraft.server.model.NCEnhanceManager
import org.apache.nlpcraft.server.tx.NCTxManager
import org.apache.nlpcraft.server.user.NCUserManager
@@ -123,7 +123,7 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
},
() ⇒ {
NCProbeManager.start(span)
- NCSuggestionsManager.start(span)
+ NCEnhanceManager.start(span)
},
() ⇒ NCFeedbackManager.start(span)
)
@@ -148,7 +148,7 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
NCRestManager,
NCQueryManager,
NCFeedbackManager,
- NCSuggestionsManager,
+ NCEnhanceManager,
NCProbeManager,
NCCompanyManager,
NCUserManager,
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/suggestion/NCSuggestionsManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCEnhanceManager.scala
similarity index 95%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/server/suggestion/NCSuggestionsManager.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCEnhanceManager.scala
index 17aa6fc..a9561a0 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/suggestion/NCSuggestionsManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCEnhanceManager.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.server.suggestion
+package org.apache.nlpcraft.server.model
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.{CopyOnWriteArrayList, CountDownLatch, TimeUnit}
@@ -43,7 +43,7 @@ import scala.collection._
/**
* TODO:
*/
-object NCSuggestionsManager extends NCService {
+object NCEnhanceManager extends NCService {
// For context word server requests.
private final val DFLT_LIMIT: Int = 20
private final val MAX_LIMIT: Int = 10000
@@ -118,18 +118,22 @@ object NCSuggestionsManager extends NCService {
}
/**
- *
+ * TODO:
* @param mdlId Model ID.
* @param minScore Context word server minimal suggestion score (default
DFLT_MIN_SCORE).
- * Increase it for suggestions count increasing, decrease it to be more
precise. Range 0 ... 2.
+ * Increase it for suggestions count increasing, decrease it to be more
precise. Range 0 ... 1.
*
* @param parent Parent.
*/
@throws[NCE]
- def suggest(mdlId: String, minScore: Option[Double], parent: Span = null):
Map[String, Seq[NCSuggestion]] =
+ def enhance(mdlId: String, minScore: Option[Double], parent: Span = null):
Map[String, Seq[NCEnhanceSuggestion]] =
startScopedSpan(
"suggest", parent, "modelId" → mdlId, "minScore" →
minScore.getOrElse(() ⇒ null)
) { _ ⇒
+ val minScoreVal = minScore.getOrElse(DFLT_MIN_SCORE)
+
+ require(minScoreVal >= 0 && minScoreVal <= 1)
+
val url = s"${Config.urlOpt.getOrElse(throw new NCE("Context word
server is not configured"))}/suggestions"
val mdl = NCProbeManager.getModel(mdlId)
@@ -249,12 +253,14 @@ object NCSuggestionsManager extends NCService {
val post = new HttpPost(url)
post.setHeader("Content-Type", "application/json")
+
post.setEntity(
new StringEntity(
GSON.toJson(
RestRequest(
sentences = batch.map(p ⇒
RestRequestSentence(p.sentence, Seq(p.index).asJava)).asJava,
- min_score =
minScore.getOrElse(DFLT_MIN_SCORE),
+ // ContextWord server range is (0, 2),
input range is (0, 1)
+ min_score = minScoreVal * 2,
// If minScore defined, we set big
limit value and in fact only minimal score
// is taken into account. Otherwise -
default value.
limit = if (minScore.isDefined)
MAX_LIMIT else DFLT_LIMIT
@@ -300,7 +306,7 @@ object NCSuggestionsManager extends NCService {
val nonEmptySuggs = allSuggs.asScala.map(p ⇒ p._1 →
p._2.asScala).filter(_._2.nonEmpty)
- val res = mutable.HashMap.empty[String,
mutable.ArrayBuffer[NCSuggestion]]
+ val res = mutable.HashMap.empty[String,
mutable.ArrayBuffer[NCEnhanceSuggestion]]
nonEmptySuggs.
foreach { case (elemId, elemSuggs) ⇒
@@ -324,14 +330,14 @@ object NCSuggestionsManager extends NCService {
res.get(elemId) match {
case Some(seq) ⇒ seq
case None ⇒
- val buf =
mutable.ArrayBuffer.empty[NCSuggestion]
+ val buf =
mutable.ArrayBuffer.empty[NCEnhanceSuggestion]
res += elemId → buf
buf
}
- seq += NCSuggestion(sugg.word, sugg.score, cnt,
sumFactor)
+ seq += NCEnhanceSuggestion(sugg.word, sugg.score,
cnt, sumFactor)
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/suggestion/NCSuggestion.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCEnhanceSuggestion.scala
similarity index 89%
rename from
nlpcraft/src/main/scala/org/apache/nlpcraft/server/suggestion/NCSuggestion.scala
rename to
nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCEnhanceSuggestion.scala
index 7ea0d28..d9041a9 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/suggestion/NCSuggestion.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/model/NCEnhanceSuggestion.scala
@@ -15,17 +15,17 @@
* limitations under the License.
*/
-package org.apache.nlpcraft.server.suggestion
+package org.apache.nlpcraft.server.model
/**
* TODO:
- * @param suggestion
+ * @param synonym
* @param ctxWorldServerScore
* @param suggestedCount
* @param totalScore
*/
-case class NCSuggestion(
- suggestion: String,
+case class NCEnhanceSuggestion(
+ synonym: String,
ctxWorldServerScore: Double,
suggestedCount: Int,
totalScore: Double
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
index b50aa66..7eca741 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/opencensus/NCOpenCensusServerStats.scala
@@ -31,7 +31,7 @@ import io.opencensus.stats._
trait NCOpenCensusServerStats {
val M_ASK_LATENCY_MS: MeasureLong = MeasureLong.create("ask_latency", "The
latency of '/ask' REST call", "ms")
val M_CHECK_LATENCY_MS: MeasureLong = MeasureLong.create("check_latency",
"The latency of '/check' REST call", "ms")
- val M_SUGGESTION_LATENCY_MS: MeasureLong =
MeasureLong.create("suggestion_latency", "The latency of '/suggestion' REST
call", "ms")
+ val M_MODEL_ENHANCE_LATENCY_MS: MeasureLong =
MeasureLong.create("model_enhance_latency", "The latency of '/model/enhance'
REST call", "ms")
val M_CANCEL_LATENCY_MS: MeasureLong =
MeasureLong.create("cancel_latency", "The latency of '/cancel' REST call", "ms")
val M_SIGNIN_LATENCY_MS: MeasureLong =
MeasureLong.create("signin_latency", "The latency of '/signin' REST call", "ms")
val M_SIGNOUT_LATENCY_MS: MeasureLong =
MeasureLong.create("signout_latency", "The latency of '/signout' REST call",
"ms")
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
index 0c6a34c..1f22354 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/rest/NCBasicRestApi.scala
@@ -37,7 +37,7 @@ import org.apache.nlpcraft.server.mdo.{NCQueryStateMdo,
NCUserMdo}
import org.apache.nlpcraft.server.opencensus.NCOpenCensusServerStats
import org.apache.nlpcraft.server.probe.NCProbeManager
import org.apache.nlpcraft.server.query.NCQueryManager
-import org.apache.nlpcraft.server.suggestion.NCSuggestionsManager
+import org.apache.nlpcraft.server.model.NCEnhanceManager
import org.apache.nlpcraft.server.user.NCUserManager
import spray.json.DefaultJsonProtocol._
import spray.json.{JsValue, RootJsonFormat}
@@ -54,17 +54,17 @@ import akka.http.scaladsl.coding.Gzip
class NCBasicRestApi extends NCRestApi with LazyLogging with NCOpenCensusTrace
with NCOpenCensusServerStats {
protected final val GSON = new Gson()
protected final val URL_VALIDATOR = new UrlValidator(Array("http",
"https"), UrlValidator.ALLOW_LOCAL_URLS)
-
+
final val API_VER = 1
final val API = "api" / s"v$API_VER"
-
+
/** */
private final val CORS_HDRS = List(
`Access-Control-Allow-Origin`.*,
`Access-Control-Allow-Credentials`(true),
`Access-Control-Allow-Headers`("Authorization", "Content-Type",
"X-Requested-With")
)
-
+
/*
* General control exception.
* Note that these classes must be public because scala 2.11 internal
errors (compilations problems).
@@ -235,7 +235,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
throw InvalidUserId(userId)
Some(userId)
-
+
case None ⇒ None
}
@@ -342,7 +342,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
startScopedSpan("signin$", "email" → req.email) { span ⇒
checkLength("email", req.email, 64)
checkLength("passwd", req.passwd, 64)
-
+
NCUserManager.signin(
req.email,
req.passwd,
@@ -385,22 +385,22 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
implicit val reqFmt: RootJsonFormat[Req] = jsonFormat1(Req)
implicit val resFmt: RootJsonFormat[Res] = jsonFormat1(Res)
-
+
entity(as[Req]) { req ⇒
startScopedSpan("signout$", "acsTok" → req.acsTok) { span ⇒
checkLength("acsTok", req.acsTok, 256)
-
+
authenticate(req.acsTok)
-
+
NCUserManager.signout(req.acsTok, span)
-
+
complete {
Res(API_OK)
}
}
}
}
-
+
/**
*
* @param reqJs
@@ -437,9 +437,9 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
checkLength("mdlId", mdlId, 32)
checkLengthOpt("data", data, 512000)
checkLengthOpt("userExtId", data, 64)
-
+
val connUser = authenticate(acsTok)
-
+
NCQueryManager.futureAsk(
getUserId(connUser, usrIdOpt, usrExtIdOpt),
txt,
@@ -483,7 +483,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
implicit val reqFmt: RootJsonFormat[Req] = jsonFormat7(Req)
implicit val resFmt: RootJsonFormat[Res] = jsonFormat2(Res)
-
+
entity(as[Req]) { req ⇒
//noinspection GetOrElseNull
startScopedSpan(
@@ -497,17 +497,17 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
checkLengthOpt("userExtId", req.usrExtId, 64)
checkLength("txt", req.txt, 1024)
checkLength("mdlId", req.mdlId, 32)
-
+
val dataJsOpt =
req.data match {
case Some(data) ⇒ Some(data.compactPrint)
case None ⇒ None
}
-
+
checkLengthOpt("data", dataJsOpt, 512000)
-
+
val connUser = authenticate(req.acsTok)
-
+
optionalHeaderValueByName("User-Agent") { usrAgent ⇒
extractClientIP { rmtAddr ⇒
val newSrvReqId = NCQueryManager.asyncAsk(
@@ -520,7 +520,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
req.enableLog.getOrElse(false),
span
)
-
+
complete {
Res(API_OK, newSrvReqId)
}
@@ -529,7 +529,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
}
}
}
-
+
/**
*
* @return
@@ -557,7 +557,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
"srvReqIds" → req.srvReqIds.getOrElse(Nil).mkString(",")) {
span ⇒
checkLength("acsTok", req.acsTok, 256)
checkLengthOpt("userExtId", req.usrExtId, 64)
-
+
val connUser = authenticate(req.acsTok)
val srvReqs = getRequests(connUser, req.srvReqIds, req.usrId,
req.usrExtId, span)
@@ -570,7 +570,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
}
}
}
-
+
/**
*
* @return
@@ -596,7 +596,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
) { span ⇒
checkLength("acsTok", req.acsTok, 256)
checkLengthOpt("userExtId", req.usrExtId, 64)
-
+
val connUser = authenticate(req.acsTok)
val states =
@@ -612,7 +612,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
)
.asJava
)
-
+
complete(
HttpResponse(
entity = HttpEntity(ContentTypes.`application/json`,
js)
@@ -626,7 +626,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
*
* @return
*/
- protected def suggestion$(): Route = {
+ protected def modelEnhance$(): Route = {
case class Req(
acsTok: String,
mdlId: String,
@@ -634,7 +634,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
)
case class Suggestion(
- suggestion: String,
+ synonym: String,
score: Double,
suggestedCount: Int
)
@@ -650,13 +650,14 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
entity(as[Req]) { req ⇒
startScopedSpan(
- "check$",
+ "modelEnhance$",
"mdlId" → req.mdlId,
"minScore" → req.minScore.getOrElse(() ⇒ null),
"acsTok" → req.acsTok
) { span ⇒
checkLength("acsTok", req.acsTok, 256)
checkLength("mdlId", req.mdlId, 32)
+ checkRangeOpt("score", req.minScore, 0, 1)
val admin = authenticateAsAdmin(req.acsTok)
@@ -664,9 +665,9 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
throw new NCE(s"Probe not found for model: ${req.mdlId}")
val res: Map[String, Seq[Suggestion]] =
- NCSuggestionsManager.suggest(req.mdlId, req.minScore,
span).
+ NCEnhanceManager.enhance(req.mdlId, req.minScore, span).
map { case (elemId, suggs) ⇒
- elemId → suggs.map(p ⇒ Suggestion(p.suggestion,
p.ctxWorldServerScore, p.suggestedCount))
+ elemId → suggs.map(p ⇒ Suggestion(p.synonym,
p.ctxWorldServerScore, p.suggestedCount))
}.toMap
complete {
@@ -703,11 +704,11 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
"usrId" → req.usrId.getOrElse(null)) { span ⇒
checkLength("acsTok", req.acsTok, 256)
checkLengthOpt("usrExtId", req.usrExtId, 64)
-
+
val connUser = authenticate(req.acsTok)
NCProbeManager.clearConversation(getUserId(connUser,
req.usrId, req.usrExtId), req.mdlId, span)
-
+
complete {
Res(API_OK)
}
@@ -742,9 +743,9 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
"usrId" → req.usrId.getOrElse(null)) { span ⇒
checkLength("acsTok", req.acsTok, 256)
checkLengthOpt("userExtId", req.usrExtId, 64)
-
+
val connUser = authenticate(req.acsTok)
-
+
NCProbeManager.clearDialog(getUserId(connUser, req.usrId,
req.usrExtId), req.mdlId, span)
complete {
@@ -753,7 +754,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
}
}
}
-
+
/**
*
* @return
@@ -800,10 +801,10 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
checkLength("adminFirstName", req.adminFirstName, 64)
checkLength("adminLastName", req.adminLastName, 64)
checkLengthOpt("adminAvatarUrl", req.adminAvatarUrl, 512000)
-
+
// Via REST only administrators of already created companies
can create new companies.
authenticateAsAdmin(req.acsTok)
-
+
val res = NCCompanyManager.addCompany(
req.name,
req.website,
@@ -819,14 +820,14 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
req.adminAvatarUrl,
span
)
-
+
complete {
Res(API_OK, res.token, res.adminId)
}
}
}
}
-
+
/**
*
* @return
@@ -846,21 +847,21 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
address: Option[String],
postalCode: Option[String]
)
-
+
implicit val reqFmt: RootJsonFormat[Req] = jsonFormat1(Req)
implicit val resFmt: RootJsonFormat[Res] = jsonFormat9(Res)
-
+
entity(as[Req]) { req ⇒
startScopedSpan("company$get", "acsTok" → req.acsTok) { span ⇒
checkLength("acsTok", req.acsTok, 256)
-
+
val connUser = authenticate(req.acsTok)
-
+
val company = NCCompanyManager.getCompany(connUser.companyId,
span) match {
case Some(c) ⇒ c
case None ⇒ throw InvalidOperation(s"Failed to find
company with ID: ${connUser.companyId}")
}
-
+
complete {
Res(API_OK,
company.id,
@@ -912,9 +913,9 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
checkLengthOpt("city", req.city, 512)
checkLengthOpt("address", req.address, 512)
checkLengthOpt("postalCode", req.postalCode, 32)
-
+
val admin = authenticateAsAdmin(req.acsTok)
-
+
NCCompanyManager.updateCompany(
admin.companyId,
req.name,
@@ -926,7 +927,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
req.postalCode,
span
)
-
+
complete {
Res(API_OK)
}
@@ -1781,7 +1782,7 @@ class NCBasicRestApi extends NCRestApi with LazyLogging
with NCOpenCensusTrace w
path(API / "signout") {
withLatency(M_SIGNOUT_LATENCY_MS, signout$) } ~ {
path(API / "cancel") {
withLatency(M_CANCEL_LATENCY_MS, cancel$) } ~
path(API / "check") { withLatency(M_CHECK_LATENCY_MS,
check$) } ~
- path(API / "suggestion") {
withLatency(M_SUGGESTION_LATENCY_MS, suggestion$) } ~
+ path(API / "model"/ "enhance") {
withLatency(M_MODEL_ENHANCE_LATENCY_MS, modelEnhance$) } ~
path(API / "clear"/ "conversation") {
withLatency(M_CLEAR_CONV_LATENCY_MS, clear$Conversation) } ~
path(API / "clear"/ "dialog") {
withLatency(M_CLEAR_DIALOG_LATENCY_MS, clear$Dialog) } ~
path(API / "company"/ "add") {
withLatency(M_COMPANY_ADD_LATENCY_MS, company$Add) } ~
diff --git a/openapi/nlpcraft_swagger.yml b/openapi/nlpcraft_swagger.yml
index 7cba9f6..10a42eb 100644
--- a/openapi/nlpcraft_swagger.yml
+++ b/openapi/nlpcraft_swagger.yml
@@ -158,14 +158,14 @@ paths:
description: Failed operation.
schema:
$ref: '#/definitions/Error'
- /suggestion:
+ /model/enhance:
post:
tags:
- - Suggestion
- summary: Synonyms suggestions for model.
+ - Model
+ summary: Model enchance suggestions for model.
description: >-
TODO:
- operationId: Suggestion
+ operationId: enhance
parameters:
- in: body
name: Payload body
@@ -189,7 +189,7 @@ paths:
type: number
format: double
description: >-
- Minimal score. Range from 0 to 2. Default 0.
+ Minimal score. Range from 0 to 1. Default 0.
responses:
'200':
description: Successful operation.
