This is an automated email from the ASF dual-hosted git repository.
aradzinski pushed a commit to branch NLPCRAFT-296
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-296 by this push:
new 3cf8e8d WIP.
3cf8e8d is described below
commit 3cf8e8dad404f0a90ade4aeb03cc9fc0203519d2
Author: unknown <[email protected]>
AuthorDate: Wed May 26 21:18:35 2021 -0700
WIP.
---
.../org/apache/nlpcraft/common/ascii/NCAsciiTable.scala | 8 ++++++++
.../org/apache/nlpcraft/common/nlp/NCNlpSentence.scala | 7 ++++---
.../apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala | 10 +++++-----
.../apache/nlpcraft/common/nlp/NCNlpSentenceToken.scala | 4 ++--
.../nlpcraft/common/nlp/NCNlpSentenceTokenBuffer.scala | 10 +++++-----
.../scala/org/apache/nlpcraft/common/util/NCUtils.scala | 4 +---
.../probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 16 +++++++---------
.../nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala | 6 +++---
.../server/nlp/enrichers/NCServerEnrichmentManager.scala | 3 +--
.../server/nlp/enrichers/basenlp/NCBaseNlpEnricher.scala | 4 ++--
.../server/nlp/enrichers/numeric/NCNumericEnricher.scala | 4 ++--
.../nlp/enrichers/stopword/NCStopWordGenerator.scala | 10 ++++++----
.../server/nlp/preproc/NCPreProcessManager.scala | 2 --
.../nlpcraft/server/nlp/wordnet/NCWordNetManager.scala | 2 +-
14 files changed, 47 insertions(+), 43 deletions(-)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/ascii/NCAsciiTable.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/ascii/NCAsciiTable.scala
index 8ef30eb..52051eb 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/ascii/NCAsciiTable.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/ascii/NCAsciiTable.scala
@@ -719,6 +719,14 @@ object NCAsciiTable {
def apply(hdrs: Any*): NCAsciiTable = new NCAsciiTable #= (hdrs: _*)
/**
+ * Creates new ASCII table with given header cells.
+ *
+ * @param hdrs Header.
+ * @return Newly created ASCII table.
+ */
+ def apply(hdrs: mutable.Seq[_]): NCAsciiTable = new NCAsciiTable #=
(hdrs.toSeq: _*)
+
+ /**
* Creates new ASCII table with given headers and data.
*
* @param hdrs Headers.
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
index b18bfbc..f508745 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentence.scala
@@ -18,10 +18,11 @@
package org.apache.nlpcraft.common.nlp
import org.apache.nlpcraft.common._
+
import java.io.{Serializable => JSerializable}
import java.util.{Collections, List => JList}
-import scala.collection.JavaConverters._
-import scala.collection.{Map, Seq, Set, mutable}
+import scala.collection.mutable
+import scala.jdk.CollectionConverters.CollectionHasAsScala
import scala.language.implicitConversions
object NCNlpSentence {
@@ -80,7 +81,7 @@ class NCNlpSentence(
*
* @param noteType Note type.
*/
- def getNotes(noteType: String): Seq[NCNlpSentenceNote] =
this.flatMap(_.getNotes(noteType)).distinct
+ def getNotes(noteType: String): Seq[NCNlpSentenceNote] =
this.flatMap(_.getNotes(noteType)).toSeq.distinct
/**
* Utility method that removes note with given ID from all tokens in this
sentence.
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
index 10b8081..6522b18 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceNote.scala
@@ -20,10 +20,10 @@ package org.apache.nlpcraft.common.nlp
import org.apache.nlpcraft.common.U
import org.apache.nlpcraft.common.ascii._
-import scala.collection.JavaConverters._
-import scala.collection.{Seq, Set, mutable}
-import scala.language.implicitConversions
import java.io.{Serializable => JSerializable}
+import scala.collection.mutable
+import scala.language.implicitConversions
+import scala.jdk.CollectionConverters.{CollectionHasAsScala, SeqHasAsJava}
/**
* Sentence token note is a typed map of KV pairs.
@@ -39,8 +39,8 @@ class NCNlpSentenceNote(private val values: Map[String,
JSerializable]) extends
lazy val noteType: String = values("noteType").asInstanceOf[String]
lazy val tokenFrom: Int = values("tokMinIndex").asInstanceOf[Int] // First
index.
lazy val tokenTo: Int = values("tokMaxIndex").asInstanceOf[Int] // Last
index.
- lazy val tokenIndexes: Seq[Int] =
values("tokWordIndexes").asInstanceOf[java.util.List[Int]].asScala // Includes
1st and last indices too.
- lazy val wordIndexes: Seq[Int] =
values("wordIndexes").asInstanceOf[java.util.List[Int]].asScala // Includes 1st
and last indices too.
+ lazy val tokenIndexes: Seq[Int] =
values("tokWordIndexes").asInstanceOf[java.util.List[Int]].asScala.toSeq //
Includes 1st and last indices too.
+ lazy val wordIndexes: Seq[Int] =
values("wordIndexes").asInstanceOf[java.util.List[Int]].asScala.toSeq //
Includes 1st and last indices too.
lazy val sparsity: Int = values("sparsity").asInstanceOf[Int]
lazy val isDirect: Boolean = values("direct").asInstanceOf[Boolean]
lazy val isUser: Boolean = {
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceToken.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceToken.scala
index 0015951..e54ecd1 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceToken.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceToken.scala
@@ -19,8 +19,8 @@ package org.apache.nlpcraft.common.nlp
import org.apache.nlpcraft.common.nlp.pos._
-import scala.collection.JavaConverters._
import scala.collection.mutable
+import scala.jdk.CollectionConverters.IterableHasAsScala
import scala.language.implicitConversions
/**
@@ -41,7 +41,7 @@ case class NCNlpSentenceToken(
def origText: String = getNlpValue[String]("origText")
def words: Int = origText.split(" ").length
def wordLength: Int = getNlpValue[Int]("wordLength").intValue()
- def wordIndexes: Seq[Int] =
getNlpValue[java.util.List[Int]]("wordIndexes").asScala
+ def wordIndexes: Seq[Int] =
getNlpValue[java.util.List[Int]]("wordIndexes").asScala.toSeq
def pos: String = getNlpValue[String]("pos")
def posDesc: String = getNlpValue[String]( "posDesc")
def lemma: String = getNlpValue[String]("lemma")
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceTokenBuffer.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceTokenBuffer.scala
index 5ecf24b..a3d1156 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceTokenBuffer.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/NCNlpSentenceTokenBuffer.scala
@@ -18,7 +18,7 @@
package org.apache.nlpcraft.common.nlp
import scala.collection.mutable.ArrayBuffer
-import scala.collection.{Seq, IndexedSeq => IdxSeq}
+import scala.collection.{Seq, IndexedSeq}
import scala.language.implicitConversions
/**
@@ -32,8 +32,8 @@ class NCNlpSentenceTokenBuffer(val tokens:
ArrayBuffer[NCNlpSentenceToken] = new
/** Stem-based hashcode. */
lazy val stemsHash: Int = stems.hashCode()
- type SSOT = IdxSeq[IdxSeq[Option[NCNlpSentenceToken]]]
- type SST = IdxSeq[IdxSeq[NCNlpSentenceToken]]
+ type SSOT = IndexedSeq[IndexedSeq[Option[NCNlpSentenceToken]]]
+ type SST = IndexedSeq[IndexedSeq[NCNlpSentenceToken]]
/**
* Gets all sequential permutations of tokens in this NLP sentence.
@@ -89,13 +89,13 @@ class NCNlpSentenceTokenBuffer(val tokens:
ArrayBuffer[NCNlpSentenceToken] = new
def permutations(toks: Seq[NCNlpSentenceToken]): SST = {
def multiple(seq: SSOT, t: NCNlpSentenceToken): SSOT =
if (seq.isEmpty)
- if (t.isStopWord) IdxSeq(IdxSeq(Some(t)), IdxSeq(None))
else IdxSeq(IdxSeq(Some(t)))
+ if (t.isStopWord) IndexedSeq(IndexedSeq(Some(t)),
IndexedSeq(None)) else IndexedSeq(IndexedSeq(Some(t)))
else {
(for (subSeq <- seq) yield subSeq :+ Some(t)) ++
(if (t.isStopWord) for (subSeq <- seq) yield subSeq :+
None else Seq.empty)
}
- var res: SSOT = IdxSeq.empty
+ var res: SSOT = IndexedSeq.empty
for (t <- toks)
res = multiple(res, t)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 8670f2c..3e7b046 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -52,9 +52,7 @@ import org.apache.nlpcraft.common.version.NCVersion
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
-import java.net.http.HttpClient
-import java.net.http.HttpRequest
-import java.net.http.HttpResponse
+import java.net.http._
import scala.annotation.tailrec
import scala.collection.mutable
import scala.concurrent.duration._
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
index c593ce2..b84a944 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
@@ -18,7 +18,6 @@
package org.apache.nlpcraft.probe.mgrs.nlp.enrichers.sort
import java.io.Serializable
-
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.makro.NCMacroParser
@@ -27,9 +26,8 @@ import org.apache.nlpcraft.common.nlp.{NCNlpSentence,
NCNlpSentenceNote, NCNlpSe
import org.apache.nlpcraft.probe.mgrs.NCProbeModel
import org.apache.nlpcraft.probe.mgrs.nlp.NCProbeEnricher
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.{Map, Seq, mutable}
+import scala.collection.mutable
+import scala.jdk.CollectionConverters.IterableHasAsJava
/**
* Sort enricher.
@@ -132,7 +130,7 @@ object NCSortEnricher extends NCProbeEnricher {
/**
*
*/
- private def validate() {
+ private def validate(): Unit = {
// Not duplicated.
require(sort.size + by.size + order.size == (sort ++ by ++
order.map(_._1)).distinct.size)
@@ -458,18 +456,18 @@ object NCSortEnricher extends NCProbeEnricher {
case Some(m) =>
if (!matches.exists(_.isSubCase(m)) &&
!m.intersect(restricted)) {
def addNotes(
- params: ArrayBuffer[(String, Any)],
+ params: mutable.ArrayBuffer[(String, Any)],
seq: Seq[NoteData],
notesName: String,
idxsName: String
- ): ArrayBuffer[(String, Any)] = {
+ ): mutable.ArrayBuffer[(String, Any)] = {
params += notesName -> seq.map(_.note).asJava
params += idxsName ->
seq.map(_.indexes.asJava).asJava
params
}
- def mkNote(params: ArrayBuffer[(String, Any)]):
Unit = {
+ def mkNote(params: mutable.ArrayBuffer[(String,
Any)]): Unit = {
val note =
NCNlpSentenceNote(m.main.map(_.index), TOK_ID, params: _*)
if (!notes.exists(n =>
ns.notesEqualOrSimilar(n, note))) {
@@ -493,7 +491,7 @@ object NCSortEnricher extends NCProbeEnricher {
if (m.subjSeq.nonEmpty)
for (subj <- m.subjSeq) {
- def addSubj(): ArrayBuffer[(String, Any)] =
+ def addSubj():
mutable.ArrayBuffer[(String, Any)] =
addNotes(mkParams(), subj,
"subjnotes", "subjindexes")
if (m.bySeq.nonEmpty)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
index 4c45982..1ff8df0 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/sentence/NCSentenceManager.scala
@@ -27,8 +27,8 @@ import org.apache.nlpcraft.model.NCModel
import java.io.{Serializable => JSerializable}
import java.util
import java.util.{List => JList}
-import scala.collection.JavaConverters.{asScalaBufferConverter, _}
-import scala.collection.{Map, Seq, mutable}
+import scala.collection.mutable
+import scala.jdk.CollectionConverters.CollectionHasAsScala
import scala.language.implicitConversions
/**
@@ -290,7 +290,7 @@ object NCSentenceManager extends NCService {
* @param ns Sentence.
* @param userNoteTypes Notes types.
*/
- private def fixIndexes(ns: NCNlpSentence, userNoteTypes: Seq[String]) {
+ private def fixIndexes(ns: NCNlpSentence, userNoteTypes: Seq[String]):
Unit = {
// Replaces other notes indexes.
for (t <- userNoteTypes :+ "nlpcraft:nlp"; note <- ns.getNotes(t)) {
val toks = ns.filter(_.contains(note)).sortBy(_.index)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
index c41c743..4f91bc2 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/NCServerEnrichmentManager.scala
@@ -36,7 +36,6 @@ import
org.apache.nlpcraft.server.nlp.enrichers.quote.NCQuoteEnricher
import org.apache.nlpcraft.server.nlp.enrichers.stopword.NCStopWordEnricher
import org.apache.nlpcraft.server.nlp.preproc.NCPreProcessManager
-import scala.collection.Seq
import scala.concurrent.ExecutionContext
import scala.util.control.Exception.catching
@@ -207,7 +206,7 @@ object NCServerEnrichmentManager extends NCService with
NCIgniteInstance {
(x._1 * 100) + x._2.indexOf(hdr.noteName)
})
- val tbl = NCAsciiTable(headers.map(_.header): _*)
+ val tbl = NCAsciiTable(headers.map(_.header))
/**
*
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/basenlp/NCBaseNlpEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/basenlp/NCBaseNlpEnricher.scala
index fb4fc1f..86f66f7 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/basenlp/NCBaseNlpEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/basenlp/NCBaseNlpEnricher.scala
@@ -24,7 +24,7 @@ import org.apache.nlpcraft.common.nlp.{NCNlpSentence,
NCNlpSentenceNote, NCNlpSe
import org.apache.nlpcraft.server.nlp.core.{NCNlpParser, NCNlpServerManager}
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
-import scala.collection._
+import scala.collection.mutable
/**
* Base NLP enricher.
@@ -135,7 +135,7 @@ object NCBaseNlpEnricher extends NCServerEnricher {
"direct" -> true
)
- tok.add(NCNlpSentenceNote(Seq(idx), "nlpcraft:nlp", seq: _*))
+ tok.add(NCNlpSentenceNote(Seq(idx), "nlpcraft:nlp", seq.toSeq:
_*))
// Add new token to NLP sentence.
ns += tok
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
index 47bba85..e2cfb5c 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
@@ -23,7 +23,7 @@ import org.apache.nlpcraft.common.nlp._
import org.apache.nlpcraft.common.nlp.numeric._
import org.apache.nlpcraft.server.nlp.enrichers.NCServerEnricher
-import scala.collection._
+import scala.collection.mutable
/**
* Numeric enricher.
@@ -415,7 +415,7 @@ object NCNumericEnricher extends NCServerEnricher {
}
}
- val toks = ns.takeWhile(_ != num.tokens.head)
+ val toks = ns.takeWhile(_ != num.tokens.head).toSeq
process(toks)
process(toks.filter(!_.isStopWord))
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordGenerator.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordGenerator.scala
index 287dfec..64454dd 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordGenerator.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/stopword/NCStopWordGenerator.scala
@@ -20,6 +20,8 @@ package org.apache.nlpcraft.server.nlp.enrichers.stopword
import org.apache.nlpcraft.common.nlp.core.NCNlpPorterStemmer
import org.apache.nlpcraft.common.util.NCUtils
+import scala.collection.mutable
+
/**
* Generates first word sequences.
*/
@@ -187,7 +189,7 @@ object NCStopWordGenerator extends App {
"couple of"
)
- private def mkGzip(path: String, lines: Traversable[Any]): Unit = {
+ private def mkGzip(path: String, lines: Iterable[Any]): Unit = {
val p = NCUtils.mkPath(s"nlpcraft/src/main/resources/stopwords/$path")
NCUtils.mkTextFile(p, lines)
@@ -195,7 +197,7 @@ object NCStopWordGenerator extends App {
}
private[stopword] def mkNounWords(): Unit = {
- val buf = new scala.collection.mutable.ArrayBuffer[String]()
+ val buf = new mutable.ArrayBuffer[String]()
for (w1 <- NOUN_WORDS)
buf += s"$w1"
@@ -203,7 +205,7 @@ object NCStopWordGenerator extends App {
for (w1 <- NOUN_WORDS; w2 <- NOUN_WORDS2)
buf += s"$w1 $w2"
- mkGzip(NOUN_WORDS_FILE, stem(buf))
+ mkGzip(NOUN_WORDS_FILE, stem(buf.toSeq))
}
private def stem(s: String): String =
@@ -342,7 +344,7 @@ object NCStopWordGenerator extends App {
for (w0 <- DWORDS_PRE; w1 <- DWORDS; w2 <- DWORDS_SUP; w3 <- QWORDS)
buf += s"$w0 $w1 $w2 $w3"
- mkGzip(FIRST_WORDS_FILE, stem(buf))
+ mkGzip(FIRST_WORDS_FILE, stem(buf.toSeq))
}
mkFirstWords()
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/preproc/NCPreProcessManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/preproc/NCPreProcessManager.scala
index cf6ee83..eebeab8 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/preproc/NCPreProcessManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/preproc/NCPreProcessManager.scala
@@ -21,8 +21,6 @@ import io.opencensus.trace.Span
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.server.nlp.spell.NCSpellCheckManager
-import scala.collection._
-
/**
* Centralized pre-processor for raw text coming from user.
*/
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/wordnet/NCWordNetManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/wordnet/NCWordNetManager.scala
index 8f91f19..9ce163e 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/wordnet/NCWordNetManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/wordnet/NCWordNetManager.scala
@@ -60,7 +60,7 @@ object NCWordNetManager extends NCService {
else
Seq.empty
})
- ).distinct
+ ).toSeq.distinct
else
Seq.empty[String]
}
