This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/master by this push:
new 8c7b018 Numeric manager improved. Alarm model tests extended.
8c7b018 is described below
commit 8c7b01874bce8eaa757450cefd9ed6150b0357dd
Author: Sergey Kamov <[email protected]>
AuthorDate: Wed Aug 4 09:49:17 2021 +0300
Numeric manager improved. Alarm model tests extended.
---
nlpcraft-examples/alarm/pom.xml | 8 ++
.../apache/nlpcraft/examples/alarm/AlarmModel.java | 92 +++++++++++++---------
.../alarm/src/main/resources/alarm_model.json | 2 +-
.../alarm/src/main/resources/alarm_samples.txt | 13 ++-
.../alarm/src/main/resources/intents.idl | 2 +-
.../nlpcraft/examples/alarm/NCAlarmModelSpec.scala | 84 ++++++++++++++++++++
.../nlpcraft/common/nlp/numeric/NCNumeric.scala | 37 +++++----
.../common/nlp/numeric/NCNumericFuzzy.scala | 65 +++++++++++++++
.../common/nlp/numeric/NCNumericManager.scala | 49 ++++++++----
.../nlp/enrichers/numeric/NCNumericEnricher.scala | 1 -
.../scala/org/apache/nlpcraft/NCTestContext.scala | 15 ++++
11 files changed, 298 insertions(+), 70 deletions(-)
diff --git a/nlpcraft-examples/alarm/pom.xml b/nlpcraft-examples/alarm/pom.xml
index 84cb0dc..0f6af79 100644
--- a/nlpcraft-examples/alarm/pom.xml
+++ b/nlpcraft-examples/alarm/pom.xml
@@ -49,6 +49,14 @@
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>nlpcraft</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java
b/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java
index 6513e0a..0e15a02 100644
---
a/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java
+++
b/nlpcraft-examples/alarm/src/main/java/org/apache/nlpcraft/examples/alarm/AlarmModel.java
@@ -17,12 +17,23 @@
package org.apache.nlpcraft.examples.alarm;
-import org.apache.nlpcraft.model.*;
-import java.time.*;
-import java.time.format.*;
-import java.util.*;
+import org.apache.nlpcraft.model.NCIntentMatch;
+import org.apache.nlpcraft.model.NCIntentRef;
+import org.apache.nlpcraft.model.NCIntentSampleRef;
+import org.apache.nlpcraft.model.NCIntentTerm;
+import org.apache.nlpcraft.model.NCModelFileAdapter;
+import org.apache.nlpcraft.model.NCRejection;
+import org.apache.nlpcraft.model.NCResult;
+import org.apache.nlpcraft.model.NCToken;
-import static java.time.temporal.ChronoUnit.*;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import static java.time.temporal.ChronoUnit.MILLIS;
/**
* Alarm example data model.
@@ -57,23 +68,50 @@ public class AlarmModel extends NCModelFileAdapter {
NCIntentMatch ctx,
@NCIntentTerm("nums") List<NCToken> numToks
) {
- long unitsCnt = numToks.stream().map(tok ->
(String)tok.meta("num:unit")).distinct().count();
+ long ms = calculateTime(numToks);
- if (unitsCnt != numToks.size())
- throw new NCRejection("Ambiguous time units.");
+ assert ms >= 0;
+ timer.schedule(
+ new TimerTask() {
+ @Override
+ public void run() {
+ System.out.println(
+ "BEEP BEEP BEEP for: " +
ctx.getContext().getRequest().getNormalizedText() + ""
+ );
+ }
+ },
+ ms
+ );
+
+ return NCResult.text("Timer set for: " +
FMT.format(LocalDateTime.now().plus(ms, MILLIS)));
+ }
+
+ @Override
+ public void onDiscard() {
+ // Clean up when model gets discarded (e.g. during testing).
+ timer.cancel();
+ }
+
+ /**
+ * Calculates time duration for given tokens.
+ *
+ * @param numToks Tokens to calculate time duration for.
+ * @return Time duration in ms.
+ */
+ public static long calculateTime(List<NCToken> numToks) {
LocalDateTime now = LocalDateTime.now();
LocalDateTime dt = now;
-
+
for (NCToken num : numToks) {
String unit = num.meta("nlpcraft:num:unit");
-
- // Skip possible fractionals to simplify.
+
+ // Skip possible fractional to simplify.
long v = ((Double)num.meta("nlpcraft:num:from")).longValue();
-
+
if (v <= 0)
throw new NCRejection("Value must be positive: " + unit);
-
+
switch (unit) {
case "second": { dt = dt.plusSeconds(v); break; }
case "minute": { dt = dt.plusMinutes(v); break; }
@@ -82,35 +120,13 @@ public class AlarmModel extends NCModelFileAdapter {
case "week": { dt = dt.plusWeeks(v); break; }
case "month": { dt = dt.plusMonths(v); break; }
case "year": { dt = dt.plusYears(v); break; }
-
+
default:
- // It shouldn't be an assert, because 'datetime' unit can
be extended.
+ // It shouldn't be an assertion, because 'datetime' unit
can be extended outside.
throw new NCRejection("Unsupported time unit: " + unit);
}
}
-
- long ms = now.until(dt, MILLIS);
-
- assert ms >= 0;
-
- timer.schedule(
- new TimerTask() {
- @Override
- public void run() {
- System.out.println(
- "BEEP BEEP BEEP for: " +
ctx.getContext().getRequest().getNormalizedText() + ""
- );
- }
- },
- ms
- );
-
- return NCResult.text("Timer set for: " + FMT.format(dt));
- }
- @Override
- public void onDiscard() {
- // Clean up when model gets discarded (e.g. during testing).
- timer.cancel();
+ return now.until(dt, MILLIS);
}
}
diff --git a/nlpcraft-examples/alarm/src/main/resources/alarm_model.json
b/nlpcraft-examples/alarm/src/main/resources/alarm_model.json
index 3d79875..9060fde 100644
--- a/nlpcraft-examples/alarm/src/main/resources/alarm_model.json
+++ b/nlpcraft-examples/alarm/src/main/resources/alarm_model.json
@@ -29,7 +29,7 @@
"description": "Alarm token indicator.",
"synonyms": [
"{ping|buzz|wake|call|hit} {me|up|me up|_}",
- "{set|_} {my|_} {wake|wake up|_}
{alarm|timer|clock|buzzer|call} {up|_}"
+ "{set|_} {my|_} {wake|wake up|_}
{alarm|timer|clock|buzzer|call} {clock|_} {up|_}"
]
}
],
diff --git a/nlpcraft-examples/alarm/src/main/resources/alarm_samples.txt
b/nlpcraft-examples/alarm/src/main/resources/alarm_samples.txt
index 188d1d6..07ce622 100644
--- a/nlpcraft-examples/alarm/src/main/resources/alarm_samples.txt
+++ b/nlpcraft-examples/alarm/src/main/resources/alarm_samples.txt
@@ -22,4 +22,15 @@
Ping me in 3 minutes tomorrow
Buzz me in an hour and 15mins
Set my alarm for 30s
-Please, wake me up in twenty five minutes!
\ No newline at end of file
+Please, wake me up in twenty-five minutes!
+Buzz me in few minutes
+Buzz me in a couple of minutes
+Wake me up in an hour
+Buzz me in a couple of hours
+Wake me up in a bit
+Set my alarm clock for an hour and 15mins
+Buzz me in one hour and 15mins
+Buzz me in 1 hour and 15mins
+Buzz me in 1h and 15mins
+Buzz me in a day, 1h and 15mins
+Buzz me in one day, 1h and 15mins
diff --git a/nlpcraft-examples/alarm/src/main/resources/intents.idl
b/nlpcraft-examples/alarm/src/main/resources/intents.idl
index bae42ae..3c1810b 100644
--- a/nlpcraft-examples/alarm/src/main/resources/intents.idl
+++ b/nlpcraft-examples/alarm/src/main/resources/intents.idl
@@ -24,7 +24,7 @@ fragment=when
@iseq = meta_tok('nlpcraft:num:isequalcondition') // Excludes
conditional statements.
tok_id() == 'nlpcraft:num' && @type == 'datetime' && @iseq == true
- }[0,7]
+ }[1,7]
// Intents (using fragments).
intent=alarm
diff --git
a/nlpcraft-examples/alarm/src/test/java/org/apache/nlpcraft/examples/alarm/NCAlarmModelSpec.scala
b/nlpcraft-examples/alarm/src/test/java/org/apache/nlpcraft/examples/alarm/NCAlarmModelSpec.scala
new file mode 100644
index 0000000..f2f6e66
--- /dev/null
+++
b/nlpcraft-examples/alarm/src/test/java/org/apache/nlpcraft/examples/alarm/NCAlarmModelSpec.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.examples.alarm
+
+import org.apache.nlpcraft.examples.alarm.AlarmModel.calculateTime
+import org.apache.nlpcraft.model.{NCIntentRef, NCIntentTerm, NCResult, NCToken}
+import org.apache.nlpcraft.{NCTestContext, NCTestEnvironment}
+import org.junit.jupiter.api.Test
+
+import java.time.LocalDateTime
+import java.time.temporal.ChronoUnit.MILLIS
+import scala.jdk.CollectionConverters.SeqHasAsJava
+
+/**
+ * Model for alarm spec.
+ */
+class AlarmModelWrapper extends AlarmModel {
+ @NCIntentRef("alarm")
+ def onMatch(@NCIntentTerm("nums") numToks: List[NCToken]): NCResult =
+ NCResult.text(String.valueOf(calculateTime(numToks.asJava)))
+}
+
+/**
+ * Unit tests that checks that alarm model produces correct time calculations.
+ */
+@NCTestEnvironment(model = classOf[AlarmModelWrapper], startClient = true)
+class NCAlarmModelSpec extends NCTestContext {
+ // Checks with 1 second precision. Enough to be sure that calculation
result is correct.
+ private def check(req: String, expectedTime: Long): Unit =
+ checkResult(req, _.toLong, (calcTime: Long) => Math.abs(expectedTime -
calcTime) <= 1000)
+
+ @Test
+ def test(): Unit = {
+ val now = LocalDateTime.now
+
+ /**
+ *
+ * @param hours
+ * @param mins
+ * @param secs
+ * @return
+ */
+ def mkPeriod(hours: Int, mins: Int, secs: Int = 0): Long =
+
now.until(now.plusHours(hours).plusMinutes(mins).plusSeconds(secs), MILLIS)
+
+ // Fuzzy `single`.
+ check("Buzz me in a minute or two.", mkPeriod(0, 1))
+ check("Buzz me in hour or two.", mkPeriod(1, 0))
+ check("Buzz me in an hour.", mkPeriod(1, 0))
+
+ // Fuzzy `few`.
+ check("Buzz me in few minutes.", mkPeriod(0, 2))
+ check("Buzz me in one or two minutes.", mkPeriod(0, 2))
+ check("Buzz me in one or two hours.", mkPeriod(2, 0))
+ check("Buzz me in a couple of minutes.", mkPeriod(0, 2))
+
+ // Fuzzy `bit`.
+ check("Buzz me in a bit.", mkPeriod(0, 2))
+
+ // Complex periods.
+ check("Buzz me in an hour and 15mins", mkPeriod(1, 15))
+ check("Buzz me in one hour and 15mins", mkPeriod(1, 15))
+ check("Buzz me in 1 hour and 15mins", mkPeriod(1, 15))
+ check("Buzz me in 1h and 15mins", mkPeriod(1, 15))
+
+ check("Buzz me in one day, 1h and 15mins", mkPeriod(25, 15))
+ check("Buzz me in a day, 1h and 15mins", mkPeriod(25, 15))
+ }
+}
diff --git a/nlpcraft-examples/alarm/src/main/resources/intents.idl
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumeric.scala
similarity index 59%
copy from nlpcraft-examples/alarm/src/main/resources/intents.idl
copy to
nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumeric.scala
index bae42ae..75a3365 100644
--- a/nlpcraft-examples/alarm/src/main/resources/intents.idl
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumeric.scala
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,18 +15,27 @@
* limitations under the License.
*/
-// Fragments (mostly for demo purposes here).
-fragment=buzz term~{tok_id() == 'x:alarm'}
-fragment=when
- term(nums)~{
- // Demonstrating term variables.
- @type = meta_tok('nlpcraft:num:unittype')
- @iseq = meta_tok('nlpcraft:num:isequalcondition') // Excludes
conditional statements.
+package org.apache.nlpcraft.common.nlp.numeric
- tok_id() == 'nlpcraft:num' && @type == 'datetime' && @iseq == true
- }[0,7]
+import org.apache.nlpcraft.common.nlp.NCNlpSentenceToken
-// Intents (using fragments).
-intent=alarm
- fragment(buzz)
- fragment(when)
\ No newline at end of file
+/**
+ *
+ * @param name
+ * @param unitType
+ */
+case class NCNumericUnit(name: String, unitType: String)
+
+/**
+ *
+ * @param tokens
+ * @param value
+ * @param isFractional
+ * @param unit
+ */
+case class NCNumeric(
+ tokens: Seq[NCNlpSentenceToken],
+ value: Double,
+ isFractional: Boolean,
+ unit: Option[NCNumericUnit]
+)
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericFuzzy.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericFuzzy.scala
new file mode 100644
index 0000000..c8d19f1
--- /dev/null
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericFuzzy.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.common.nlp.numeric
+
+import org.apache.nlpcraft.common.makro.NCMacroParser
+
+case class NCNumericFuzzyPeriod(unit: NCNumericUnit, value: Int)
+
+/**
+ * Fuzzy numeric configuration.
+ */
+object NCNumericFuzzy {
+ val NUMS: Map[String, NCNumericFuzzyPeriod] = {
+ val parser = new NCMacroParser
+
+ def make(txt: String, unit: NCNumericUnit, value: Int): Seq[(String,
NCNumericFuzzyPeriod)] =
+ parser.expand(txt).map(_ -> NCNumericFuzzyPeriod(unit, value))
+
+ import org.apache.nlpcraft.common.nlp.numeric.{NCNumericUnit => U}
+
+ val singleDt =
+ Map(
+ "in {a|an|_} second {or two|_}" -> U("second", "datetime"),
+ "in {a|an|_} hour {or two|_}" -> U("hour", "datetime"),
+ "in {a|an|_} minute {or two|_}" -> U("minute", "datetime"),
+ "in {a|an|_} day {or two|_}" -> U("day", "datetime"),
+ "in {a|an|_} week {or two|_}" -> U("week", "datetime"),
+ "in {a|an|_} month {or two|_}" -> U("month", "datetime"),
+ "in {a|an|_} year {or two|_}" -> U("year", "datetime")
+ ).flatMap { case (txt, u) => make(txt, u, 1) }
+
+ val bitDt =
+ Map(
+ "in {a|an} bit" -> U("minute", "datetime")
+ ).flatMap { case (txt, u) => make(txt, u, 2) }
+
+ val fewDt =
+ Map(
+ "in {a|an|_} {few|couple of|one or two|two or three} seconds"
-> U("second", "datetime"),
+ "in {a|an|_} {few|couple of|one or two|two or three} minutes"
-> U("minute", "datetime"),
+ "in {a|an|_} {few|couple of|one or two|two or three} hours" ->
U("hour", "datetime"),
+ "in {a|an|_} {few|couple of|one or two|two or three} days" ->
U("day", "datetime"),
+ "in {a|an|_} {few|couple of|one or two|two or three} weeks" ->
U("week", "datetime"),
+ "in {a|an|_} {few|couple of|one or two|two or three} months"
-> U("month", "datetime"),
+ "in {a|an|_} {few|couple of|one or two|two or three} years" ->
U("year", "datetime")
+ ).flatMap { case (txt, u) => make(txt, u, 2) }
+
+ singleDt ++ bitDt ++ fewDt
+ }
+}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala
index e20a18b..a428ab9 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/nlp/numeric/NCNumericManager.scala
@@ -17,26 +17,15 @@
package org.apache.nlpcraft.common.nlp.numeric
-import java.text.{DecimalFormat, ParseException}
-import java.util.Locale
-
import io.opencensus.trace.Span
import org.apache.nlpcraft.common.NCService
import org.apache.nlpcraft.common.nlp._
import org.apache.nlpcraft.common.nlp.core.NCNlpCoreManager
import org.apache.nlpcraft.common.util.NCUtils.mapResource
-case class NCNumericUnit(name: String, unitType: String)
-case class NCNumeric(
- tokens: Seq[NCNlpSentenceToken],
- value: Double,
- isFractional: Boolean,
- unit: Option[NCNumericUnit]
-)
+import java.text.{DecimalFormat, ParseException}
+import java.util.Locale
-/**
- * Numeric detection manager.
- */
object NCNumericManager extends NCService {
// Sets EN numeric format.
Locale.setDefault(Locale.forLanguageTag("EN"))
@@ -119,6 +108,34 @@ object NCNumericManager extends NCService {
}
/**
+ * Tries to find special cases for numerics definition, without explicit
numerics.
+ *
+ * Example: 'in an hour'.
+ *
+ * @param ns Sentence.
+ */
+ private def findFuzzy(ns: NCNlpSentence): Seq[NCNumeric] = {
+ val senToks: Seq[NCNlpSentenceToken] = ns.tokens.toSeq
+ val senWords: Seq[String] = senToks.map(_.normText)
+
+ NCNumericFuzzy.NUMS.map { case (txt, period) => txt.split(" ") ->
period }.flatMap {
+ case (dtWords, dtPeriod) =>
+ senWords.indexOfSlice(dtWords) match {
+ case -1 => None
+ case idx =>
+ Some(
+ NCNumeric(
+ tokens = senToks.slice(idx, idx +
dtWords.length),
+ value = dtPeriod.value,
+ isFractional = false,
+ unit = Some(dtPeriod.unit)
+ )
+ )
+ }
+ }
+ }
+
+ /**
*
* @param parent Optional parent span.
*/
@@ -434,7 +451,11 @@ object NCNumericManager extends NCService {
val usedToks = nums.flatMap(_.tokens)
- (nums ++ ns.filter(t =>
!usedToks.contains(t)).flatMap(mkSolidNumUnit)).sortBy(_.tokens.head.index).distinct
+ (
+ nums ++
+ ns.filter(t => !usedToks.contains(t)).flatMap(mkSolidNumUnit)
++
+ findFuzzy(ns)
+ ).sortBy(_.tokens.head.index).distinct
}
}
}
diff --git
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
index e2cfb5c..b28f198 100644
---
a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
+++
b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/nlp/enrichers/numeric/NCNumericEnricher.scala
@@ -438,7 +438,6 @@ object NCNumericEnricher extends NCServerEnricher {
num.tokens.foreach(_.add(note))
}
-
}
}
}
\ No newline at end of file
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
index 0201315..d0b3ceb 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/NCTestContext.scala
@@ -165,6 +165,21 @@ abstract class NCTestContext {
assertEquals(expResp, resExtractor(res.getResult.get))
}
+ /**
+ *
+ * @param req
+ * @param resExtractor
+ * @param validator
+ * @tparam T
+ */
+ protected def checkResult[T](req: String, resExtractor: String => T,
validator: T => Boolean): Unit = {
+ val res = getClient.ask(req)
+
+ assertTrue(res.isOk, s"Unexpected result,
error=${res.getResultError.orElse(null)}")
+ assertTrue(res.getResult.isPresent)
+ assertTrue(validator.apply(resExtractor(res.getResult.get)))
+ }
+
final protected def getClient: NCTestClient = {
if (cli == null)
throw new IllegalStateException("Client is not started.")
