This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-30
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/NLPCRAFT-30 by this push:
new 1d47f2f WIP.
1d47f2f is described below
commit 1d47f2fdfcbabed7dc929a16daaf1a018546fedf
Author: Sergey Kamov <[email protected]>
AuthorDate: Mon Apr 20 17:30:57 2020 +0300
WIP.
---
.../nlpcraft/examples/sql/SqlModelTest.scala | 58 +++++++++++++++
.../tools/sqlgen/impl/NCSqlExtractorImpl.scala | 84 ++++++++++++++--------
.../mgrs/nlp/enrichers/sort/NCSortEnricher.scala | 11 +--
.../nlp/enrichers/sort/NCEnricherSortSpec.scala | 8 +++
4 files changed, 128 insertions(+), 33 deletions(-)
diff --git a/src/main/scala/org/apache/nlpcraft/examples/sql/SqlModelTest.scala
b/src/main/scala/org/apache/nlpcraft/examples/sql/SqlModelTest.scala
index b36169e..d82e758 100644
--- a/src/main/scala/org/apache/nlpcraft/examples/sql/SqlModelTest.scala
+++ b/src/main/scala/org/apache/nlpcraft/examples/sql/SqlModelTest.scala
@@ -447,6 +447,64 @@ class SqlModelTest {
| LEFT JOIN shippers ON orders.ship_via =
shippers.shipper_id
| LEFT JOIN employees ON orders.employee_id =
employees.employee_id
|ORDER BY
+ | orders.shipped_date DESC
+ |LIMIT
+ | 1000
+ """.stripMargin
+ ),
+ Case(
+ Seq(
+ "give me the orders sorted by ship date asc"
+ ),
+ """SELECT
+ | orders.shipped_date,
+ | orders.order_id,
+ | orders.order_date,
+ | orders.required_date,
+ | customers.customer_id,
+ | customers.company_name,
+ | customers.contact_name,
+ | employees.employee_id,
+ | employees.last_name,
+ | employees.first_name,
+ | shippers.shipper_id,
+ | shippers.company_name,
+ | shippers.phone
+ |FROM
+ | orders
+ | LEFT JOIN customers ON orders.customer_id =
customers.customer_id
+ | LEFT JOIN shippers ON orders.ship_via =
shippers.shipper_id
+ | LEFT JOIN employees ON orders.employee_id =
employees.employee_id
+ |ORDER BY
+ | orders.shipped_date ASC
+ |LIMIT
+ | 1000
+ """.stripMargin
+ ),
+ Case(
+ // Default sort (nlpcraft:sort shouldn't be found)
+ Seq(
+ "give me the orders sorted by date"
+ ),
+ """SELECT
+ | orders.order_date,
+ | orders.order_id,
+ | orders.required_date,
+ | customers.customer_id,
+ | customers.company_name,
+ | customers.contact_name,
+ | employees.employee_id,
+ | employees.last_name,
+ | employees.first_name,
+ | shippers.shipper_id,
+ | shippers.company_name,
+ | shippers.phone
+ |FROM
+ | orders
+ | LEFT JOIN customers ON orders.customer_id =
customers.customer_id
+ | LEFT JOIN shippers ON orders.ship_via =
shippers.shipper_id
+ | LEFT JOIN employees ON orders.employee_id =
employees.employee_id
+ |ORDER BY
| orders.order_id DESC
|LIMIT
| 1000
diff --git
a/src/main/scala/org/apache/nlpcraft/model/tools/sqlgen/impl/NCSqlExtractorImpl.scala
b/src/main/scala/org/apache/nlpcraft/model/tools/sqlgen/impl/NCSqlExtractorImpl.scala
index e070119..30fd764 100644
---
a/src/main/scala/org/apache/nlpcraft/model/tools/sqlgen/impl/NCSqlExtractorImpl.scala
+++
b/src/main/scala/org/apache/nlpcraft/model/tools/sqlgen/impl/NCSqlExtractorImpl.scala
@@ -112,10 +112,10 @@ class NCSqlExtractorImpl(schema: NCSqlSchema, variant:
NCVariant) extends NCSqlE
/**
*
* @param link
- * @param typ
+ * @param element
* @return
*/
- private def findColumn(link: NCToken, typ: String): NCSqlColumn =
+ private def findColumn(link: NCToken, element: String): NCSqlColumn =
findAnyColumnTokenOpt(link) match {
// If reference is column - sort by column.
case Some(t) ⇒ extractColumn(t)
@@ -123,7 +123,7 @@ class NCSqlExtractorImpl(schema: NCSqlSchema, variant:
NCVariant) extends NCSqlE
// If reference is table - sort by any PK column of table.
findAnyTableTokenOpt(link) match {
case Some(tab) ⇒ tab.getColumns.asScala.minBy(col ⇒ if
(col.isPk) 0 else 1)
- case None ⇒ throw new NCException(s"Unexpected $typ link:
$link")
+ case None ⇒ throw new NCException(s"Unexpected $element
link: $link")
}
}
@@ -149,18 +149,44 @@ class NCSqlExtractorImpl(schema: NCSqlSchema, variant:
NCVariant) extends NCSqlE
* @param tok
* @return
*/
- private def getLinks(
- variant: NCVariant, tok: NCToken, idxField: String, noteField: String,
canBeEmpty: Boolean
+ private def getOptionalLinks(variant: NCVariant, tok: NCToken, idxsField:
String, notesField: String): Seq[NCToken] = {
+ val idxsOpt: Option[util.List[Integer]] =
tok.metaOpt(s"${tok.getId}:$idxsField").asScala
+
+ idxsOpt match {
+ case Some(idxs) ⇒
+ val notes: util.List[String] =
tok.metax(s"${tok.getId}:$notesField")
+
+ idxs.asScala.map(idx ⇒ {
+ if (idx < variant.size) {
+ val link = variant.get(idx)
+
+ if (!notes.contains(link.getId))
+ throw new NCException(s"Unexpected token with
index: $idx, type: ${link.getId}")
+
+ link
+ }
+ else
+ throw new NCException(s"Token not found with index:
$idx")
+ })
+ case None ⇒ Seq.empty
+ }
+ }
+
+ /**
+ *
+ * @param variant
+ * @param tok
+ * @return
+ */
+ private def getSingleMandatoryLinks(
+ variant: NCVariant, tok: NCToken, idxField: String, noteField: String
): Seq[NCToken] = {
- val idxsOpt: Option[util.List[Integer]] =
tok.metaOpt(s"${tok.getId}:$idxField").asScala
+ val idxs: util.List[Integer] =
tok.metaOpt(s"${tok.getId}:$idxField").asScala.getOrElse(throw new
NCException(s"Empty indexes for: $tok"))
- if (idxsOpt.isEmpty && !canBeEmpty)
- throw new NCException(s"Empty indexes for: $tok")
+ val note: String = tok.metax(s"${tok.getId}:$noteField")
- idxsOpt.get.asScala.map(idx ⇒ {
+ idxs.asScala.map(idx ⇒
if (idx < variant.size) {
- val note: String = tok.metax(s"${tok.getId}:$noteField")
-
val link = variant.get(idx)
if (link.getId != note)
@@ -170,9 +196,10 @@ class NCSqlExtractorImpl(schema: NCSqlSchema, variant:
NCVariant) extends NCSqlE
}
else
throw new NCException(s"Token not found with index: $idx")
- })
+ )
}
+
/**
*
* @param tok
@@ -194,7 +221,7 @@ class NCSqlExtractorImpl(schema: NCSqlSchema, variant:
NCVariant) extends NCSqlE
override def extractLimit(limitTok: NCToken): NCSqlLimit = {
checkTokenId(limitTok, "nlpcraft:limit")
- val links = getLinks(variant, limitTok, "indexes", "note", canBeEmpty
= false)
+ val links = getSingleMandatoryLinks(variant, limitTok, "indexes",
"note")
links.size match {
case 1 ⇒
@@ -217,33 +244,34 @@ class NCSqlExtractorImpl(schema: NCSqlSchema, variant:
NCVariant) extends NCSqlE
override def extractSort(sortTok: NCToken): util.List[NCSqlSort] = {
checkTokenId(sortTok, "nlpcraft:sort")
- val tables = getLinks(variant, sortTok, "subjindexes", "subjnotes",
canBeEmpty = false).
+ val tables = getOptionalLinks(variant, sortTok, "subjindexes",
"subjnotes").
map(link ⇒ findTable(link, "SORT"))
- val cols = getLinks(variant, sortTok, "byindexes", "bynotes",
canBeEmpty = true).
+ val cols = getOptionalLinks(variant, sortTok, "byindexes", "bynotes").
map(link ⇒ findColumn(link, "SORT BY"))
val asc = getAsc(sortTok, "nlpcraft:sort:asc", dflt = false)
- require(tables.nonEmpty)
+ require(tables.nonEmpty || cols.nonEmpty)
- tables.flatMap(t ⇒ {
- var colTabs = cols.filter(_.getTable == t.getTable)
+ def getSorts(cols: Seq[NCSqlColumn]): Seq[NCSqlSort] = cols.map(col ⇒
NCSqlSortImpl(col, asc))
- if (colTabs.isEmpty)
- colTabs = t.getColumns.asScala.filter(_.isPk)
+ if (tables.nonEmpty)
+ tables.flatMap(t ⇒ {
+ var colTabs = cols.filter(_.getTable == t.getTable)
- if (colTabs.isEmpty)
- colTabs = t.getColumns.asScala.take(1)
+ if (colTabs.isEmpty)
+ colTabs = t.getColumns.asScala.filter(_.isPk)
- require(colTabs.nonEmpty)
+ if (colTabs.isEmpty)
+ colTabs = t.getColumns.asScala.take(1)
- colTabs.map(col ⇒ {
- val sort: NCSqlSort = NCSqlSortImpl(col, asc)
+ require(colTabs.nonEmpty)
- sort
- })
- }).asJava
+ getSorts(colTabs)
+ }).asJava
+ else
+ getSorts(cols).asJava
}
/**
diff --git
a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
index 993dd45..e5b91ea 100644
---
a/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
+++
b/src/main/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCSortEnricher.scala
@@ -204,8 +204,8 @@ object NCSortEnricher extends NCProbeEnricher {
between.isEmpty || between.forall(p ⇒ p.isStopWord ||
p.stem == STEM_AND)
}
- val minIdx = toks.dropWhile(_.isNlp).head.index
- val maxIdx = toks.reverse.dropWhile(_.isNlp).head.index
+ val minIdx = toks.dropWhile(!_.isUser).head.index
+ val maxIdx = toks.reverse.dropWhile(!_.isUser).head.index
require(minIdx <= maxIdx)
@@ -229,9 +229,10 @@ object NCSortEnricher extends NCProbeEnricher {
if (res.isEmpty && !nullable)
throw new AssertionError(s"Invalid null result " +
- s"[tokensTexts=[${toks.map(_.origText).mkString(", ")}]" +
- s", tokensIndexes=[${toks.map(_.index).mkString(", ")}]" +
- s", allData=[${toksNoteData.mkString(", ")}]" +
+ s"[tokensTexts=[${toks.map(_.origText).mkString("|")}]" +
+ s", notes=[${toks.flatten.map(n ⇒
s"${n.noteType}:[${n.tokenIndexes.mkString(",")}]").mkString("|")}]" +
+ s", tokensIndexes=[${toks.map(_.index).mkString("|")}]" +
+ s", allData=[${toksNoteData.mkString("|")}]" +
s"]"
)
diff --git
a/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCEnricherSortSpec.scala
b/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCEnricherSortSpec.scala
index 0ed4758..dd25663 100644
---
a/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCEnricherSortSpec.scala
+++
b/src/test/scala/org/apache/nlpcraft/probe/mgrs/nlp/enrichers/sort/NCEnricherSortSpec.scala
@@ -181,6 +181,14 @@ class NCEnricherSortSpec extends NCEnricherBaseSpec {
nlp(text = ",", isStop = true),
usr(text = "B", id = "B"),
nlp(text = "from bottom up order", isStop = true)
+ ),
+ _ ⇒ checkExists(
+ "organize by A, B the descending",
+ srt(text = "organize by", byNotes = Seq("A", "B"), byIndexes =
Seq(1, 3), asc = Some(false)),
+ usr(text = "A", id = "A"),
+ nlp(text = ",", isStop = true),
+ usr(text = "B", id = "B"),
+ nlp(text = "the descending", isStop = true)
)
)
}
