This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-108
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-108 by this push:
new 8d6f778 WIP.
8d6f778 is described below
commit 8d6f7789d9339b21526e279398392675555b4612
Author: Aaron Radzinski <[email protected]>
AuthorDate: Sat Oct 3 13:58:07 2020 -0700
WIP.
---
nlpcraft/src/main/resources/nlpcraft.conf | 86 +++++++++++-----------
.../nlpcraft/model/tools/cmdline/NCCli.scala | 2 +-
.../model/tools/cmdline/NCCliServerBeacon.scala | 24 +++++-
.../org/apache/nlpcraft/probe/NCProbeBoot.scala | 3 +-
.../probe/mgrs/nlp/NCProbeEnrichmentManager.scala | 5 +-
.../org/apache/nlpcraft/server/NCServer.scala | 59 ++++++++++++---
.../nlpcraft/server/query/NCQueryManager.scala | 2 +-
.../org/apache/nlpcraft/server/sql/NCSql.scala | 14 +---
8 files changed, 122 insertions(+), 73 deletions(-)
diff --git a/nlpcraft/src/main/resources/nlpcraft.conf
b/nlpcraft/src/main/resources/nlpcraft.conf
index 6993c5d..9edc0eb 100644
--- a/nlpcraft/src/main/resources/nlpcraft.conf
+++ b/nlpcraft/src/main/resources/nlpcraft.conf
@@ -209,6 +209,49 @@ nlpcraft {
ctxword.url="http://localhost:5000";
}
+ # Basic NLP toolkit to use on both server and probes. Possible values:
+ # - 'opennlp'
+ # - 'stanford'
+ #
+ # NOTE: Stanford CoreNLP requires special installation due to its
licensing.
+ # See https://nlpcraft.apache.org/integrations.html#stanford for more
details.
+ nlpEngine = "opennlp"
+
+ # External configuration resources.
+ #
+ # NOTE:
+ # ----
+ # Due to licensing restrictions of the official ASF release policy some of
the
+ # configuration for NLPCraft cannot be shipped with the official Apache
release.
+ # Instead, NLPCraft will attempt to download these configuration files
from the
+ # external URL upon the first start.
+ #
+ # NLPCraft will attempt to download the missing configuration files from
URL defined
+ # in 'nlpcraft.extConfig.extUrl' property and place them into
'nlpcraft.extConfig.locDir'
+ # folder on the local file system. On subsequent starts, NLPCraft will
check if the required
+ # file is already present locally and skip the download in such case. If
'nlpcraft.extConfig.checkMd5'
+ # property is set to 'true' then on each start NLPCraft will check the
checksum of each file
+ # locally and remote and will re-download such file if the MD5 checksums
don't match.
+ #
+ # By default, the external configuration is stored in the main Git
repository for NLPCraft
+ # project from where it will be downloaded ('/external' folder). See this
folder in the Git
+ # repository for more information:
https://github.com/apache/incubator-nlpcraft/tree/master/external
+ extConfig {
+ # Mandatory.
+ extUrl =
"https://github.com/apache/incubator-nlpcraft/raw/master/external";
+
+ # Optional.
+ # Default value is $USER_HOME/.nlpcraft/extcfg
+ # locDir = ...
+
+ # If 'true', on each start NLPCraft will check the MD5 checksum of the
each local and remote
+ # external configuration file and will re-download such file if the
checksum doesn't match.
+ # Set it to 'false' to speed up the bootstrap of the NLPCraft server
and the data probe if you
+ # are certain that all external configuration files are properly
downloaded and available
+ # in 'nlpcraft.extConfig.locDir' local folder.
+ checkMd5 = true
+ }
+
# +---------------------+
# | Probe configuration. |
# +---------------------+
@@ -282,47 +325,4 @@ nlpcraft {
# When exceeded the request will be automatically rejected.
resultMaxSizeBytes = 1048576
}
-
- # Basic NLP toolkit to use on both server and probes. Possible values:
- # - 'opennlp'
- # - 'stanford'
- #
- # NOTE: Stanford CoreNLP requires special installation due to its
licensing.
- # See https://nlpcraft.apache.org/integrations.html#stanford for more
details.
- nlpEngine = "opennlp"
-
- # External configuration resources.
- #
- # NOTE:
- # ----
- # Due to licensing restrictions of the official ASF release policy some of
the
- # configuration for NLPCraft cannot be shipped with the official Apache
release.
- # Instead, NLPCraft will attempt to download these configuration files
from the
- # external URL upon the first start.
- #
- # NLPCraft will attempt to download the missing configuration files from
URL defined
- # in 'nlpcraft.extConfig.extUrl' property and place them into
'nlpcraft.extConfig.locDir'
- # folder on the local file system. On subsequent starts, NLPCraft will
check if the required
- # file is already present locally and skip the download in such case. If
'nlpcraft.extConfig.checkMd5'
- # property is set to 'true' then on each start NLPCraft will check the
checksum of each file
- # locally and remote and will re-download such file if the MD5 checksums
don't match.
- #
- # By default, the external configuration is stored in the main Git
repository for NLPCraft
- # project from where it will be downloaded ('/external' folder). See this
folder in the Git
- # repository for more information:
https://github.com/apache/incubator-nlpcraft/tree/master/external
- extConfig {
- # Mandatory.
- extUrl =
"https://github.com/apache/incubator-nlpcraft/raw/master/external";
-
- # Optional.
- # Default value is $USER_HOME/.nlpcraft/extcfg
- # locDir = ...
-
- # If 'true', on each start NLPCraft will check the MD5 checksum of the
each local and remote
- # external configuration file and will re-download such file if the
checksum doesn't match.
- # Set it to 'false' to speed up the bootstrap of the NLPCraft server
and the data probe if you
- # are certain that all external configuration files are properly
downloaded and available
- # in 'nlpcraft.extConfig.locDir' local folder.
- checkMd5 = true
- }
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
index 8203cb9..ed40344 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCli.scala
@@ -829,7 +829,7 @@ object NCCli extends App {
val tbl = new NCAsciiTable
tbl += ("PID", s"${g(beacon.pid)}")
- tbl += ("JDBC URL", s"${g(beacon.jdbcUrl)}")
+ tbl += ("JDBC URL", s"${g(beacon.dbUrl)}")
tbl += ("REST endpoint", s"${g(beacon.restEndpoint)}")
tbl += ("Uplink", s"${g(beacon.upLink)}")
tbl += ("Downlink", s"${g(beacon.downLink)}")
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliServerBeacon.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliServerBeacon.scala
index c31ad6d..335f8f1 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliServerBeacon.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/cmdline/NCCliServerBeacon.scala
@@ -20,17 +20,39 @@ package org.apache.nlpcraft.model.tools.cmdline
/**
*
* @param pid
+ * @param dbUrl
+ * @param dbDriver
+ * @param dbPoolMin
+ * @param dbPoolMax
+ * @param dbPoolInit
+ * @param dbPoolInc
+ * @param initDb
* @param restEndpoint
* @param upLink
* @param downLink
+ * @param startMs
+ * @param nlpEngine
+ * @param tokenProviders
+ * @param extConfigUrl
+ * @param ph
*/
case class NCCliServerBeacon(
pid: Long,
- jdbcUrl: String,
+ dbUrl: String,
+ dbDriver: String,
+ dbPoolMin: Int,
+ dbPoolMax: Int,
+ dbPoolInit: Int,
+ dbPoolInc: Int,
+ dbInit: Boolean,
restEndpoint: String,
upLink: String,
downLink: String,
startMs: Long,
+ nlpEngine: String,
+ tokenProviders: String,
+ extConfigUrl: String,
+ filePath: String,
@transient var ph: ProcessHandle = null
)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
index 0ff93a8..e5c11e9 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/NCProbeBoot.scala
@@ -49,6 +49,7 @@ import
org.apache.nlpcraft.probe.mgrs.nlp.enrichers.stopword.NCStopWordEnricher
import
org.apache.nlpcraft.probe.mgrs.nlp.enrichers.suspicious.NCSuspiciousNounsEnricher
import org.apache.nlpcraft.probe.mgrs.nlp.validate.NCValidateManager
import org.apache.nlpcraft.common.ansi.NCAnsi._
+import org.apache.nlpcraft.common._
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -385,7 +386,7 @@ private [probe] object NCProbeBoot extends LazyLogging with
NCOpenCensusTrace {
tbl.margin(bottom = 1, top = 1)
- tbl += s"Probe started $ansiBlueFg$dur$ansiReset"
+ tbl += s"Probe started ${b(dur)}"
tbl.info(logger)
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
index b8d27a9..69d6c26 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/NCProbeEnrichmentManager.scala
@@ -26,7 +26,6 @@ import java.util.{Date, Objects}
import io.opencensus.trace.{Span, Status}
import org.apache.nlpcraft.common.NCErrorCodes._
import org.apache.nlpcraft.common._
-import org.apache.nlpcraft.common.ansi.NCAnsi._
import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.common.debug.NCLogHolder
@@ -231,7 +230,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
val tbl = NCAsciiTable()
- tbl += (s"${b("Text")}", nlpSens.map(s ⇒ g(s.text)))
+ tbl += (s"${b("Text")}", nlpSens.map(s ⇒ rv(s.text)))
tbl += (s"${b("Model ID")}", mdlId)
tbl += (s"${b("User ID")}", usrId)
tbl += (s"${b(" First Name")}", senMeta.getOrElse("FIRST_NAME", ""))
@@ -243,7 +242,7 @@ object NCProbeEnrichmentManager extends NCService with
NCOpenCensusModelStats {
tbl += (s"${b("User Agent")}", senMeta.getOrElse("USER_AGENT", ""))
tbl += (s"${b("Remote Address")}", senMeta.getOrElse("REMOTE_ADDR",
""))
tbl += (s"${b("Server Timestamp")}", new
Date(java.lang.Long.parseLong(senMeta("RECEIVE_TSTAMP").toString)))
- tbl += (s"${b("Server Request ID")}", srvReqId)
+ tbl += (s"${b("Server Request ID")}", rv(g(srvReqId)))
logger.info(s"New request received from server:\n$tbl")
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
index c37a6dc..5709e5f 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/NCServer.scala
@@ -152,7 +152,7 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
tbl.margin(top = 1, bottom = 1)
- tbl += s"Server started $ansiBlueFg$dur$ansiReset"
+ tbl += s"Server started ${b(dur)}"
tbl.info(logger)
}
@@ -195,8 +195,6 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
asciiLogo()
- storeBeacon()
-
val lifecycle = new CountDownLatch(1)
catching(classOf[Throwable]) either startManagers() match {
@@ -208,6 +206,9 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
System.exit(1)
case _ ⇒ // Managers started OK.
+ // Store beacon file once all managers started OK.
+ storeBeacon()
+
ackStart()
Runtime.getRuntime.addShutdownHook(new Thread() {
@@ -239,21 +240,42 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
final object Config extends NCConfigurable {
final private val pre = "nlpcraft.server"
+ lazy val pid = ProcessHandle.current().pid()
lazy val restHost = getString(s"$pre.rest.host")
lazy val restPort = getInt(s"$pre.rest.port")
lazy val upLink = getString(s"$pre.probe.links.upLink")
lazy val downLink = getString(s"$pre.probe.links.downLink")
- lazy val jdbcUrl = getString(s"$pre.database.jdbc.url")
+ lazy val dbUrl = getString(s"$pre.database.jdbc.url")
+ lazy val dbDriver = getString(s"$pre.database.jdbc.driver")
+ lazy val dbPoolMin =
getInt(s"$pre.database.c3p0.pool.minSize")
+ lazy val dbPoolMax =
getInt(s"$pre.database.c3p0.pool.maxSize")
+ lazy val dbPoolInit =
getInt(s"$pre.database.c3p0.pool.initSize")
+ lazy val dbPoolInc =
getInt(s"$pre.database.c3p0.pool.acquireIncrement")
+ lazy val dbInit = getBool(s"$pre.database.igniteDbInitialize")
+ lazy val tokProviders = getString(s"$pre.tokenProviders")
+ lazy val nlpEngine = getString("nlpcraft.nlpEngine")
+ lazy val extCfgUrl = getString("nlpcraft.extConfig.extUrl")
+ lazy val restEndpoint =
s"${Config.restHost}:${Config.restPort}"
}
try {
managed(new ObjectOutputStream(new FileOutputStream(path)))
acquireAndGet { stream ⇒
stream.writeObject(NCCliServerBeacon(
- pid = ProcessHandle.current().pid(),
- jdbcUrl = Config.jdbcUrl,
- restEndpoint =
s"${Config.restHost}:${Config.restPort}",
+ pid = Config.pid,
+ dbUrl = Config.dbUrl,
+ dbDriver = Config.dbDriver,
+ dbPoolMin = Config.dbPoolMin,
+ dbPoolMax = Config.dbPoolMax,
+ dbPoolInit = Config.dbPoolInit,
+ dbPoolInc = Config.dbPoolInc,
+ dbInit = Config.dbInit,
+ restEndpoint = Config.restEndpoint,
upLink = Config.upLink,
downLink = Config.downLink,
+ tokenProviders = Config.tokProviders,
+ nlpEngine = Config.nlpEngine,
+ extConfigUrl = Config.extCfgUrl,
+ filePath = path.getAbsolutePath,
startMs = currentTime
))
stream.flush()
@@ -262,7 +284,25 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
// Make sure beacon is deleted when server process exits.
path.deleteOnExit()
- logger.info(s"Server beacon saved: ${path.getAbsolutePath}")
+ val tbl = NCAsciiTable()
+
+ tbl += (s"${b("PID")}", Config.pid)
+ tbl += (s"${b("Database URL")}", Config.dbUrl)
+ tbl += (s"${b(" Driver")}", Config.dbDriver)
+ tbl += (s"${b(" Pool min")}", Config.dbPoolMin)
+ tbl += (s"${b(" Pool init")}", Config.dbPoolInit)
+ tbl += (s"${b(" Pool max")}", Config.dbPoolMax)
+ tbl += (s"${b(" Pool increment")}", Config.dbPoolInc)
+ tbl += (s"${b(" Reset on start")}", Config.dbInit)
+ tbl += (s"${b("REST endpoint")}", Config.restEndpoint)
+ tbl += (s"${b("Probe uplink")}", Config.upLink)
+ tbl += (s"${b("Probe downlink")}", Config.downLink)
+ tbl += (s"${b("Token providers")}", Config.tokProviders)
+ tbl += (s"${b("NLP engine")}", Config.nlpEngine)
+ tbl += (s"${b("External config URL")}", Config.extCfgUrl)
+ tbl += (s"${b("Beacon file path")}", path.getAbsolutePath)
+
+ logger.info(s"Sever configuration:\n$tbl")
}
catch {
case e: IOException ⇒ U.prettyError(logger, "Failed to save
server beacon.", e)
@@ -274,8 +314,7 @@ object NCServer extends App with NCIgniteInstance with
LazyLogging with NCOpenCe
managed(new ObjectInputStream(new FileInputStream(path)))
acquireAndGet { _.readObject() }
} match {
case Left(e) ⇒
- U.prettyError(logger, s"Failed to read existing server
beacon: ${path.getAbsolutePath}", e)
-
+ logger.trace(s"Failed to read existing server beacon:
${path.getAbsolutePath}", e)
logger.trace(s"Overriding failed server beacon:
${path.getAbsolutePath}")
save()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
index 9ffbce5..7082c0e 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/query/NCQueryManager.scala
@@ -269,7 +269,7 @@ object NCQueryManager extends NCService with
NCIgniteInstance with NCOpenCensusS
tbl += (s"${b("Model ID")}", mdlId)
tbl += (s"${b("Agent")}", usrAgent.getOrElse("<n/a>"))
tbl += (s"${b("Remote Address")}", rmtAddr.getOrElse("<n/a>"))
- tbl += (s"${b("Server Request ID")}", srvReqId)
+ tbl += (s"${b("Server Request ID")}", rv(g(srvReqId)))
// TODO: need to pretty print data JSON
// tbl += (s"${ansiBlueFg}Data$ansiReset", data.getOrElse(""))
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sql/NCSql.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sql/NCSql.scala
index 520d820..585f9ed 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sql/NCSql.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/sql/NCSql.scala
@@ -25,7 +25,6 @@ import com.typesafe.scalalogging.LazyLogging
import org.apache.ignite.{Ignite, IgniteAtomicSequence, IgniteJdbcThinDriver}
import org.apache.ignite.transactions.Transaction
import org.apache.nlpcraft.common._
-import org.apache.nlpcraft.common.ascii.NCAsciiTable
import org.apache.nlpcraft.common.config.NCConfigurable
import org.apache.nlpcraft.server.tx.NCTxManager
import resource._
@@ -121,18 +120,7 @@ object NCSql extends LazyLogging {
ds
}
- val tbl = new NCAsciiTable
-
- tbl += ("JDBC URL", Config.url)
- tbl += ("JDBC driver", Config.driver)
- tbl += ("C3P0 pool", s"" +
- s"min=${b(Config.minPoolSize)}, " +
- s"ini=${b(Config.initPoolSize)}, " +
- s"max=${b(Config.maxPoolSize)}, " +
- s"inc=${b(Config.acqInc)}"
- )
-
- logger.info(s"Data source initialized:\n${tbl.toString}")
+ logger.trace(s"Data source initialized.")
/**
* Wraps database error.
