This is an automated email from the ASF dual-hosted git repository. sergeykamov pushed a commit to branch NLPCRAFT-147 in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 0368ee63cd0f251fa4c107e3fbf44021e381d811 Author: Sergey Kamov <[email protected]> AuthorDate: Thu Oct 8 15:59:33 2020 +0300 NCIntentSample annotation refactored as repeatable. --- .../org/apache/nlpcraft/model/NCIntentSample.java | 3 +- .../nlpcraft/model/NCIntentSampleRepeatable.java | 37 +++++++++++++++ .../test/impl/NCTestAutoModelValidatorImpl.scala | 53 +++++++++++----------- .../apache/nlpcraft/probe/mgrs/NCProbeModel.scala | 2 +- .../probe/mgrs/deploy/NCDeployManager.scala | 42 +++++++++++------ .../apache/nlpcraft/model/NCIntentSampleSpec.scala | 49 ++++++++++++++++++++ 6 files changed, 142 insertions(+), 44 deletions(-) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSample.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSample.java index 2433867..f6452bb 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSample.java +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSample.java @@ -44,9 +44,10 @@ import static java.lang.annotation.RetentionPolicy.*; * @see NCModel#onMatchedIntent(NCIntentMatch) * @see NCTestAutoModelValidator */ -@Documented +// @Documented // TODO? Documented is not valid here @Retention(value=RUNTIME) @Target(value=METHOD) +@Repeatable(NCIntentSampleRepeatable.class) public @interface NCIntentSample { /** * Gets a list of user input samples that should match corresponding intent. This annotation should be diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSampleRepeatable.java b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSampleRepeatable.java new file mode 100644 index 0000000..6c65eba --- /dev/null +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/NCIntentSampleRepeatable.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nlpcraft.model; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.METHOD; + +@Retention(RetentionPolicy.RUNTIME) +@Target(value=METHOD) +/** + * TODO: + */ +public @interface NCIntentSampleRepeatable { + /** + * TODO: + * @return + */ + NCIntentSample[] value(); +} diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala index 607e216..2f2c1e9 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/model/tools/test/impl/NCTestAutoModelValidatorImpl.scala @@ -66,7 +66,7 @@ private [test] object NCTestAutoModelValidatorImpl extends LazyLogging { * @param samples * @return */ - private def process(samples: Map[/*Model ID*/String, Map[String/*Intent ID*/, Seq[String]/*Samples*/]]): Boolean = { + private def process(samples: Map[/*Model ID*/String, Map[String/*Intent ID*/, Seq[Seq[String]]/*Samples*/]]): Boolean = { case class Result( modelId: String, intentId: String, @@ -74,37 +74,36 @@ private [test] object NCTestAutoModelValidatorImpl extends LazyLogging { pass: Boolean, error: Option[String] ) - + val results = samples.flatMap { case (mdlId, samples) ⇒ - val cli = new NCTestClientBuilder().newBuilder.build - - cli.open(mdlId) - - try { - def ask(intentId: String, txt: String): Result = { - val res = cli.ask(txt) - - if (res.isFailed) - Result(mdlId, intentId, txt, pass = false, Some(res.getResultError.get())) - else if (intentId != res.getIntentId) - Result(mdlId, intentId, txt, pass = false, Some(s"Unexpected intent ID '${res.getIntentId}'")) - else - Result(mdlId, intentId, txt, pass = true, None) + def ask(intentId: String, txts: Seq[String]): Seq[Result] = { + val cli = new NCTestClientBuilder().newBuilder.build + + try { + cli.open(mdlId) + + txts.map (txt ⇒ { + val res = cli.ask(txt) + + if (res.isFailed) + Result(mdlId, intentId, txt, pass = false, Some(res.getResultError.get())) + else if (intentId != res.getIntentId) + Result(mdlId, intentId, txt, pass = false, Some(s"Unexpected intent ID '${res.getIntentId}'")) + else + Result(mdlId, intentId, txt, pass = true, None) + }) } - - for ((intentId, seq) ← samples; txt ← seq) yield ask(intentId, txt) + finally + cli.close() } - finally - cli.close() - }.toList - - // Sort for better output. - results.sortBy(res ⇒ (res.modelId, res.intentId)) - + + for ((intentId, seq) ← samples; txts ← seq) yield ask(intentId, txts) + }.flatten.toList + val tbl = NCAsciiTable() - + tbl #= ("Model ID", "Intent ID", "+/-", "Text", "Error") - + for (res ← results) tbl += ( res.modelId, diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala index 2bca270..fae4496 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/NCProbeModel.scala @@ -45,5 +45,5 @@ case class NCProbeModel( exclStopWordsStems: Set[String], suspWordsStems: Set[String], elements: Map[String /*Element ID*/ , NCElement], - samples: Map[String, Seq[String]] + samples: Map[String, Seq[Seq[String]]] ) diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala index d9330ac..339dd27 100644 --- a/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala +++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/probe/mgrs/deploy/NCDeployManager.scala @@ -1445,7 +1445,7 @@ object NCDeployManager extends NCService with DecorateAsScala { * @param mdl Model to scan. */ @throws[NCE] - private def scanSamples(mdl: NCModel): Map[String, Seq[String]] = { + private def scanSamples(mdl: NCModel): Map[String, Seq[Seq[String]]] = { var annFound = false val mdlId = mdl.getId @@ -1453,11 +1453,14 @@ object NCDeployManager extends NCService with DecorateAsScala { mdl.getClass.getDeclaredMethods.flatMap(mtd ⇒ { def mkMethodName: String = s"${mtd.getDeclaringClass.getName}#${mtd.getName}(...)" - val smpAnn = mtd.getAnnotation(CLS_SAMPLE) + val smpAnns = mtd.getAnnotationsByType(CLS_SAMPLE) + + require(smpAnns != null) + val intAnn = mtd.getAnnotation(CLS_INTENT) val refAnn = mtd.getAnnotation(CLS_INTENT_REF) - if (smpAnn != null || intAnn != null || refAnn != null) { + if (smpAnns.nonEmpty || intAnn != null || refAnn != null) { annFound = true def mkIntentId(): String = @@ -1468,7 +1471,7 @@ object NCDeployManager extends NCService with DecorateAsScala { else throw new AssertionError() - if (smpAnn != null) { + if (smpAnns.nonEmpty) { if (intAnn == null && refAnn == null) { logger.warn(s"`@NCTestSample annotation without corresponding @NCIntent or @NCIntentRef annotations [" + s"mdlId=$mdlId, " + @@ -1478,9 +1481,9 @@ object NCDeployManager extends NCService with DecorateAsScala { None } else { - val samples = smpAnn.value().toList + val samples = smpAnns.toSeq.map(_.value().toSeq) - if (samples.isEmpty) { + if (samples.exists(_.isEmpty)) { logger.warn(s"@NCTestSample annotation is empty [" + s"mdlId=$mdlId, " + s"callback=$mkMethodName" + @@ -1488,14 +1491,16 @@ object NCDeployManager extends NCService with DecorateAsScala { None } - else if (U.containsDups(samples)) { + else if (U.containsDups(samples.flatten.toList)) { logger.warn(s"@NCTestSample annotation has duplicates [" + s"mdlId=$mdlId, " + s"callback=$mkMethodName, " + s"dups=${U.getDups(samples).mkString("'", ", ", "'")}" + s"]") - Some(mkIntentId() → samples.distinct) + // Samples is list of list. Duplicates cannot be inside one list, + // but possible between different lists. + Some(mkIntentId() → samples.map(_.distinct).distinct) } else Some(mkIntentId() → samples) @@ -1525,18 +1530,25 @@ object NCDeployManager extends NCService with DecorateAsScala { map(NCNlpPorterStemmer.stem).map(_.split(" ").toSeq). toSet + case class Case(modelId: String, sample: String) + + val processed = mutable.HashSet.empty[Case] + samples. - flatMap { case (_, samples) ⇒ samples.map(_.toLowerCase) }. + flatMap { case (_, samples) ⇒ samples.flatten.map(_.toLowerCase) }. map(s ⇒ s → SEPARATORS.foldLeft(s)((s, ch) ⇒ s.replaceAll(s"\\$ch", s" $ch "))). foreach { case (s, sNorm) ⇒ - val seq: Seq[String] = sNorm.split(" ").map(NCNlpPorterStemmer.stem) + if (processed.add(Case(mdlId, s))) { + val seq: Seq[String] = sNorm.split(" ").map(NCNlpPorterStemmer.stem) + + if (!allSyns.exists(_.intersect(seq).nonEmpty)) + logger.warn(s"@IntentSample sample doesn't contain any direct synonyms [" + + s"mdlId=$mdlId, " + + s"sample='$s'" + + s"]") + } - if (!allSyns.exists(_.intersect(seq).nonEmpty)) - logger.warn(s"@IntentSample sample doesn't contain any direct synonyms [" + - s"mdlId=$mdlId, " + - s"sample='$s'" + - s"]") } samples diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentSampleSpec.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentSampleSpec.scala new file mode 100644 index 0000000..d9872af --- /dev/null +++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/model/NCIntentSampleSpec.scala @@ -0,0 +1,49 @@ +package org.apache.nlpcraft.model + +import java.util + +import org.apache.nlpcraft.model.tools.test.NCTestAutoModelValidator +import org.junit.jupiter.api.Test + +import scala.collection.JavaConverters._ + +/** + * Sample annotation test model. + */ +class NCIntentSampleSpecModel extends NCModelAdapter( + "nlpcraft.sample.ann.model.test", "Sample annotation Test Model", "1.0" +) { + private implicit def convert(s: String): NCResult = NCResult.text(s) + + override def getElements: util.Set[NCElement] = Set(mkElement("x1"), mkElement("x2")).asJava + + private def mkElement(id: String): NCElement = + new NCElement { + override def getId: String = id + } + + @NCIntent("intent=intent1 term={id=='x1'}") + @NCIntentSample(Array("x1", "x1")) + @NCIntentSample(Array("unknown", "unknown")) + private def onX1(ctx: NCIntentMatch): NCResult = "OK" + + @NCIntentSample(Array("x1", "x2", "x3")) + @NCIntentSample(Array("x1", "x2")) + @NCIntentSample(Array("x1")) + @NCIntent("intent=intent2 term={id=='x2'}") + private def onX2(ctx: NCIntentMatch): NCResult = "OK" +} + +/** + * Sample annotation test. + */ +class NCIntentSampleSpec { + @Test + def test(): Unit = { + System.setProperty("NLPCRAFT_TEST_MODELS", "org.apache.nlpcraft.model.NCIntentSampleSpecModel") + + // Note that this validation can print validation warnings for this 'NCIntentSampleSpecModel' model. + // Its is expected behaviour because not model is tested, but validation itself. + NCTestAutoModelValidator.isValid() + } +}
