This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0f809ad9b0 Detect language: choose the languages to detect and a
faster low accu… (#9434)
0f809ad9b0 is described below
commit 0f809ad9b05210ddaab19080de1846948d34b4c6
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Nov 16 17:26:45 2026 +0100
Detect language: choose the languages to detect and a faster low accu…
(#9434)
* Detect language: choose the languages to detect and a faster low accuracy
mode, fixes #9433
* Detect language: integration test for the language choice, low accuracy
mode and blank texts, #9433
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../pages/pipeline/transforms/detectlanguage.adoc | 31 ++
.../transforms/0128-detect-language.hpl | 217 +++++++++++
.../golden-0128-detect-language-chosen.csv | 6 +
.../datasets/golden-0128-detect-language.csv | 6 +
.../transforms/main-0128-detect-language.hwf | 79 ++++
.../golden-0128-detect-language-chosen.json | 24 ++
.../dataset/golden-0128-detect-language.json | 24 ++
.../unit-test/0128-detect-language UNIT.json | 49 +++
.../transforms/language/DetectLanguage.java | 115 ++++--
.../transforms/language/DetectLanguageChoice.java | 50 +++
.../transforms/language/DetectLanguageData.java | 7 +
.../transforms/language/DetectLanguageDialog.java | 170 +++------
.../transforms/language/DetectLanguageMeta.java | 204 ++++++++++-
.../language/messages/messages_en_US.properties | 16 +
.../language/DetectLanguageAsyncTest.java | 22 +-
.../language/DetectLanguageMetaTest.java | 178 ++++++++-
.../transforms/language/DetectLanguageTest.java | 401 +++++++++++++++++++++
17 files changed, 1429 insertions(+), 170 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
index ace4d07ded..6268a61e9f 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
@@ -35,8 +35,39 @@
https://github.com/pemistahl/lingua?tab=readme-ov-file#3-which-languages-are-sup
|Option|Description
|Transform name|Name of the transform.
This name must be unique throughout the pipeline.
+|Corpus text field|The field with the text to examine.
+A variable is allowed.
+|Use Parallelism|Detect the language of several rows at the same time.
+The rows can leave the transform in another order than they came in.
+|Low accuracy mode|Use only the three-letter language models.
+This is faster and uses far less memory, but it is less accurate: short texts
of a few words are often given the wrong language.
+Use it for longer texts.
+|Languages to detect|The languages the text can be in, by their Lingua name
(`ENGLISH`, `FRENCH`, ...).
+A variable is allowed.
+Leave the list empty to detect every spoken language.
+List at least two languages.
|===
+== Output fields
+
+[options="header"]
+|===
+|Field|Description
+|detected_language|The name of the detected language, for example `ENGLISH`,
or `UNKNOWN`.
+|detected_language_confidence|Lingua's score for that language, relative to
the other languages.
+The detected language is the one with the highest score, so the value is
always 1: it is not a probability and does not tell how sure the detection is.
+Empty when the language is `UNKNOWN`.
+|===
+
+A row with an empty text passes with `UNKNOWN` and no confidence.
+The language is also `UNKNOWN` when the two most likely languages score
exactly the same.
+
+== Performance
+
+Detecting every spoken language loads the models of all 74 spoken languages:
about 1.4 GB of memory, released when the transform finishes.
+When you know which languages the data can contain, list them: with five
languages the transform is about 2.5 times as fast and needs about 175 MB.
+A text in a language that is not listed is given one of the listed languages,
or `UNKNOWN`.
+
== Installation
This transform is not part of the default Hop client.
diff --git a/integration-tests/transforms/0128-detect-language.hpl
b/integration-tests/transforms/0128-detect-language.hpl
new file mode 100644
index 0000000000..23f4337844
--- /dev/null
+++ b/integration-tests/transforms/0128-detect-language.hpl
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0128-detect-language</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Detect language with the default options (every spoken
language, high accuracy, one thread) and with a chosen set of languages, low
accuracy mode and parallel jobs. Blank and null texts pass with
UNKNOWN.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ <parameter>
+ <name>DETECT_LANGUAGE_FIELD</name>
+ <default_value>text</default_value>
+ <description>The corpus field accepts a variable</description>
+ </parameter>
+ <parameter>
+ <name>DETECT_LANGUAGE_THIRD</name>
+ <default_value>GERMAN</default_value>
+ <description>A language to choose accepts a variable</description>
+ </parameter>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>texts</from>
+ <to>detect, default options</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>texts</from>
+ <to>detect, chosen languages</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>detect, default options</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>detect, chosen languages</from>
+ <to>validate chosen</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>texts</name>
+ <type>DataGrid</type>
+ <description>English, French and German text, only spaces, and no text at
all (null).</description>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>1</item>
+ <item>The children are playing in the garden while their parents
prepare dinner in the kitchen.</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Les enfants jouent dans le jardin pendant que leurs parents
préparent le dîner dans la cuisine.</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>Die Kinder spielen im Garten, während ihre Eltern in der Küche
das Abendessen vorbereiten.</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item> </item>
+ </line>
+ <line>
+ <item>5</item>
+ <item/>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>text</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>detect, default options</name>
+ <type>DetectLanguage</type>
+ <description>Every spoken language, high accuracy, one
thread.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <corpusField>text</corpusField>
+ <parallelism>N</parallelism>
+ <lowAccuracy>N</lowAccuracy>
+ <languages>
+ </languages>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>detect, chosen languages</name>
+ <type>DetectLanguage</type>
+ <description>English, French and German only, low accuracy mode, parallel
jobs: the rows can come out in another order.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <corpusField>${DETECT_LANGUAGE_FIELD}</corpusField>
+ <parallelism>Y</parallelism>
+ <lowAccuracy>Y</lowAccuracy>
+ <languages>
+ <language>
+ <name>ENGLISH</name>
+ </language>
+ <language>
+ <name>FRENCH</name>
+ </language>
+ <language>
+ <name>${DETECT_LANGUAGE_THIRD}</name>
+ </language>
+ </languages>
+ <attributes/>
+ <GUI>
+ <xloc>336</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description></description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>560</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate chosen</name>
+ <type>Dummy</type>
+ <description></description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>560</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
new file mode 100644
index 0000000000..782fe6e881
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
@@ -0,0 +1,6 @@
+id,detected_language
+1,ENGLISH
+2,FRENCH
+3,GERMAN
+4,UNKNOWN
+5,UNKNOWN
diff --git
a/integration-tests/transforms/datasets/golden-0128-detect-language.csv
b/integration-tests/transforms/datasets/golden-0128-detect-language.csv
new file mode 100644
index 0000000000..782fe6e881
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0128-detect-language.csv
@@ -0,0 +1,6 @@
+id,detected_language
+1,ENGLISH
+2,FRENCH
+3,GERMAN
+4,UNKNOWN
+5,UNKNOWN
diff --git a/integration-tests/transforms/main-0128-detect-language.hwf
b/integration-tests/transforms/main-0128-detect-language.hwf
new file mode 100644
index 0000000000..9d1831237c
--- /dev/null
+++ b/integration-tests/transforms/main-0128-detect-language.hwf
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0128-detect-language</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0128-detect-language UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0128-detect-language UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>304</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0128-detect-language UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
new file mode 100644
index 0000000000..d9c08c0784
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0128-detect-language-chosen.csv",
+ "name": "golden-0128-detect-language-chosen",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "detected_language",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
new file mode 100644
index 0000000000..7c1acb1722
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0128-detect-language.csv",
+ "name": "golden-0128-detect-language",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "detected_language",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0128-detect-language
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json
new file mode 100644
index 0000000000..99ba566cdc
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0128-detect-language
UNIT.json
@@ -0,0 +1,49 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "detected_language",
+ "data_set_field": "detected_language"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0128-detect-language",
+ "transform_name": "validate"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "detected_language",
+ "data_set_field": "detected_language"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0128-detect-language-chosen",
+ "transform_name": "validate chosen"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0128-detect-language UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0128-detect-language.hpl"
+}
diff --git
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
index bf569b32e7..611d2aac1d 100644
---
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
+++
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
@@ -18,7 +18,6 @@
package org.apache.hop.pipeline.transforms.language;
import static com.github.pemistahl.lingua.api.Language.UNKNOWN;
-import static
com.github.pemistahl.lingua.api.LanguageDetectorBuilder.fromAllSpokenLanguages;
import static java.lang.System.arraycopy;
import static java.util.concurrent.ForkJoinPool.commonPool;
import static org.apache.commons.lang3.StringUtils.isBlank;
@@ -27,7 +26,9 @@ import static org.apache.hop.core.util.Utils.isEmpty;
import com.github.pemistahl.lingua.api.Language;
import com.github.pemistahl.lingua.api.LanguageDetector;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
import java.util.SortedMap;
import java.util.concurrent.ExecutorService;
@@ -44,6 +45,7 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
private static final Class<?> PKG = DetectLanguage.class;
private static final String
CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD =
"DetectLanguage.Exception.CouldnotFindField";
+ private static final String ERROR_CODE = "DetectLanguage001";
public DetectLanguage(
TransformMeta transformMeta,
@@ -71,15 +73,52 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
private final Object lock = new Object();
- private final LanguageDetector detector =
- fromAllSpokenLanguages().withPreloadedLanguageModels().build();
+ /** The outcome of one detection: the language name and its confidence, null
when unknown. */
+ record Detection(String language, Double confidence) {}
- private void awaitJobs() {
- awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ /**
+ * The Lingua builder for the configured options.
+ *
+ * @param languages the languages to choose from; empty means all spoken
languages
+ * @param lowAccuracy use only the trigram models
+ */
+ static LanguageDetectorBuilder createBuilder(List<Language> languages,
boolean lowAccuracy) {
+ LanguageDetectorBuilder builder =
+ languages.isEmpty()
+ ? LanguageDetectorBuilder.fromAllSpokenLanguages()
+ : LanguageDetectorBuilder.fromLanguages(languages.toArray(new
Language[0]));
+ if (lowAccuracy) {
+ builder = builder.withLowAccuracyMode();
+ }
+ return builder.withPreloadedLanguageModels();
}
- private void awaitJobs(final long timeoutMillis) {
+ @Override
+ public boolean init() {
+ if (!super.init()) {
+ return false;
+ }
+ try {
+ data.detector = createBuilder(meta.resolveLanguages(this),
meta.isLowAccuracy()).build();
+ return true;
+ } catch (Exception e) {
+ logError(BaseMessages.getString(PKG,
"DetectLanguage.Error.DetectorNotBuilt"), e);
+ return false;
+ }
+ }
+
+ /**
+ * Waits for accepted asynchronous work, at most {@link
#DRAIN_TIMEOUT_MILLIS}.
+ *
+ * @return true when every job finished, false when some were abandoned
+ */
+ private boolean awaitJobs() {
+ return awaitJobs(DRAIN_TIMEOUT_MILLIS);
+ }
+
+ private boolean awaitJobs(final long timeoutMillis) {
boolean interrupted = false;
+ boolean drained;
final long deadline = System.currentTimeMillis() + timeoutMillis;
synchronized (lock) {
long remaining = timeoutMillis;
@@ -93,7 +132,8 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
}
remaining = deadline - System.currentTimeMillis();
}
- if (jobs > 0) {
+ drained = jobs == 0;
+ if (!drained) {
// A worker is stuck and no provider timeout applies. Do not block
disposal forever: log and
// abandon the remaining jobs.
logBasic(
@@ -107,6 +147,7 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
if (interrupted) {
Thread.currentThread().interrupt();
}
+ return drained;
}
/**
@@ -150,8 +191,16 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
@Override
public void dispose() {
// The common pool is shared. Drain our work, but never shut the pool down.
- awaitJobs();
- detector.unloadLanguageModels();
+ // After an error or a stop the transform ends without finish(), and
parallel jobs can still be
+ // queued. They use the detector: unloading under them would make them
fail on a missing
+ // detector, or load the models again with nothing left to unload them.
+ boolean drained = awaitJobs();
+ if (drained && data.detector != null) {
+ // Lingua keeps its models in static maps: release what this transform
asked for. Another
+ // Detect language transform that is still running loads them again on
demand.
+ data.detector.unloadLanguageModels();
+ data.detector = null;
+ }
final boolean shouldLog;
synchronized (lock) {
// Only log a failure we have not already surfaced through the run loop,
so it is not logged
@@ -189,7 +238,9 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- if (isEmpty(meta.getCorpusField())) {
+ // The field combo accepts variables
+ data.corpusField = resolve(meta.getCorpusField());
+ if (isEmpty(data.corpusField)) {
throw new HopException(
BaseMessages.getString(PKG,
"DetectLanguage.Error.CorpusFieldMissing"));
}
@@ -198,7 +249,6 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
} // End If first
try {
- process(r);
if (isRowLevel()) {
logRowlevel(
BaseMessages.getString(
@@ -206,6 +256,7 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
"DetectLanguage.LineNumber",
getLinesRead() + " : " + getInputRowMeta().getString(r)));
}
+ process(r);
} catch (Exception e) {
// An infrastructure/worker failure must not be attributed to the
current input row.
checkFailure();
@@ -216,8 +267,10 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
}
private void process(Object[] inputRow) throws HopException {
- String corpus = trim((String) inputRow[data.indexOfCorpusField]);
+ String corpus = trim(getInputRowMeta().getString(inputRow,
data.indexOfCorpusField));
if (isBlank(corpus)) {
+ // Nothing to detect: the row passes with an unknown language and no
confidence.
+ putDetection(inputRow, new Detection(UNKNOWN.name(), null));
return;
}
@@ -230,11 +283,13 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
private void handleRowError(Object[] row, Exception e) throws HopException {
if (!getTransformMeta().isDoingErrorHandling()) {
- throw new HopException("Error processing row", e);
+ throw new HopException(
+ BaseMessages.getString(PKG,
"DetectLanguage.ErrorInTransformRunning") + e.getMessage(),
+ e);
}
// Error output has the same serialization requirement as normal output.
synchronized (this) {
- putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(),
"DetectLanguage001");
+ putError(getInputRowMeta(), row, 1, e.toString(), data.corpusField,
ERROR_CODE);
}
}
@@ -257,11 +312,18 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
() -> {
try {
try {
- processSync(text, inputRow);
+ // Stopped: nobody wants the row anymore, and dispose() waits
for the queue to
+ // drain.
+ if (!isStopped() || isSafeStopped()) {
+ processSync(text, inputRow);
+ }
} catch (Exception e) {
handleRowError(inputRow, e);
}
} catch (Throwable e) {
+ // Throwable, not Exception: nobody reads the pool's Future, so
an Error caught
+ // nowhere else would make the row disappear without a trace. An
Error says nothing
+ // about the row: it always stops the transform.
synchronized (lock) {
if (failure == null) {
failure = e;
@@ -286,16 +348,14 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
}
}
- @SuppressWarnings(
- "javabugs:S2259") // the probabilities come from the map's own keys and
are never null
private void cacheIndexPositions() throws HopException {
if (data.indexOfCorpusField < 0) {
- data.indexOfCorpusField =
data.previousRowMeta.indexOfValue(meta.getCorpusField());
+ data.indexOfCorpusField =
data.previousRowMeta.indexOfValue(data.corpusField);
if (data.indexOfCorpusField < 0) {
// The field is unreachable!
throw new HopException(
BaseMessages.getString(
- PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD,
meta.getCorpusField()));
+ PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD,
data.corpusField));
}
}
if (data.indexOfDetectedLanguage < 0) {
@@ -321,9 +381,13 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
}
}
+ /**
+ * Detects the language of a text: the most likely language, unless the
second one is as likely or
+ * closer than Lingua's minimum relative distance, in which case the
language is unknown.
+ */
@SuppressWarnings(
"javabugs:S2259") // the probabilities come from the map's own keys and
are never null
- private void processSync(String corpus, Object[] inputRow) throws
HopTransformException {
+ static Detection detect(LanguageDetector detector, String corpus) {
SortedMap<Language, Double> confidenceValues =
detector.computeLanguageConfidenceValues(corpus);
Language detectedLanguage = UNKNOWN;
Double confidence = null;
@@ -348,11 +412,18 @@ public class DetectLanguage extends
BaseTransform<DetectLanguageMeta, DetectLang
}
}
}
+ return new Detection(detectedLanguage.name(), confidence);
+ }
+
+ private void processSync(String corpus, Object[] inputRow) throws
HopTransformException {
+ putDetection(inputRow, detect(data.detector, corpus));
+ }
+ private void putDetection(Object[] inputRow, Detection detection) throws
HopTransformException {
Object[] outputRow = new Object[inputRow.length + 2 + OVER_ALLOCATE_SIZE];
arraycopy(inputRow, 0, outputRow, 0, inputRow.length);
- outputRow[data.indexOfDetectedLanguage] = detectedLanguage.name();
- outputRow[data.indexOfDetectedLanguageConfidence] = confidence;
+ outputRow[data.indexOfDetectedLanguage] = detection.language();
+ outputRow[data.indexOfDetectedLanguageConfidence] = detection.confidence();
putRow(data.outputRowMeta, outputRow);
}
diff --git
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
new file mode 100644
index 0000000000..7b48e58e34
--- /dev/null
+++
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.language;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiTableColumn;
+import org.apache.hop.core.gui.plugin.GuiTableColumnType;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+/** One language the Detect language transform may answer. */
+@Getter
+@Setter
+public class DetectLanguageChoice {
+
+ /** The Lingua language name, for example ENGLISH. A variable is allowed. */
+ @HopMetadataProperty(
+ key = "name",
+ injectionKey = "LANGUAGE",
+ injectionKeyDescription = "DetectLanguage.Injection.Language")
+ @GuiTableColumn(
+ id = "name",
+ order = "10",
+ type = GuiTableColumnType.COMBO,
+ comboValuesMethod = "languageNames",
+ label = "i18n::DetectLanguage.Column.Language.Label",
+ variables = true,
+ width = 250)
+ private String name = "";
+
+ public DetectLanguageChoice() {}
+
+ public DetectLanguageChoice(String name) {
+ this.name = name;
+ }
+}
diff --git
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
index ef5d72bf98..12f16dbfb3 100644
---
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
+++
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
@@ -17,6 +17,7 @@
package org.apache.hop.pipeline.transforms.language;
+import com.github.pemistahl.lingua.api.LanguageDetector;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
@@ -30,6 +31,12 @@ public class DetectLanguageData extends BaseTransformData
implements ITransformD
public IRowMeta outputRowMeta;
public int NrPrevFields;
+ /** The name of the corpus field, its variables resolved. */
+ public String corpusField;
+
+ /** Built in init() from the configured languages and accuracy mode. */
+ public LanguageDetector detector;
+
public DetectLanguageData() {
super();
indexOfCorpusField = -1;
diff --git
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
index c5a1fcb766..372b679cee 100644
---
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
+++
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
@@ -14,48 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hop.pipeline.transforms.language;
-import static org.apache.hop.core.util.Utils.isEmpty;
-import static org.apache.hop.i18n.BaseMessages.getString;
-import static org.eclipse.swt.SWT.BORDER;
-import static org.eclipse.swt.SWT.CHECK;
-import static org.eclipse.swt.SWT.CURSOR_WAIT;
-import static org.eclipse.swt.SWT.READ_ONLY;
-import static org.eclipse.swt.SWT.RIGHT;
-
-import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.PipelineMeta;
-import org.apache.hop.pipeline.transform.ITransformDialog;
-import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
-import org.apache.hop.ui.core.dialog.ErrorDialog;
-import org.apache.hop.ui.core.widget.ComboItems;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
+import org.apache.hop.ui.core.widget.ComboVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
-import org.eclipse.swt.SWT;
-import org.eclipse.swt.custom.CCombo;
-import org.eclipse.swt.events.FocusEvent;
-import org.eclipse.swt.events.FocusListener;
-import org.eclipse.swt.graphics.Cursor;
-import org.eclipse.swt.layout.FormAttachment;
-import org.eclipse.swt.layout.FormData;
-import org.eclipse.swt.widgets.Button;
-import org.eclipse.swt.widgets.Label;
-import org.eclipse.swt.widgets.Listener;
+import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Shell;
-public class DetectLanguageDialog extends BaseTransformDialog implements
ITransformDialog {
+public class DetectLanguageDialog extends BaseTransformDialog {
private static final Class<?> PKG = DetectLanguageDialog.class;
- private boolean gotPreviousFields = false;
-
- private CCombo wCorpusFieldName;
- private Button wParallelism;
private final DetectLanguageMeta input;
+ private GuiCompositeWidgets widgets;
public DetectLanguageDialog(
Shell parent,
@@ -69,82 +49,66 @@ public class DetectLanguageDialog extends
BaseTransformDialog implements ITransf
@Override
public String open() {
createShell(BaseMessages.getString(PKG,
"DetectLanguageDialog.Shell.Title"));
-
buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build();
- Listener lsMod = e -> input.setChanged();
changed = input.hasChanged();
- // CorpusFieldName field
- Label wlCorpusFieldName = new Label(shell, RIGHT);
- wlCorpusFieldName.setText(getString(PKG,
"DetectLanguageDialog.CorpusFieldName.Label"));
- PropsUi.setLook(wlCorpusFieldName);
- FormData fdlCorpusFieldName = new FormData();
- fdlCorpusFieldName.left = new FormAttachment(0, 0);
- fdlCorpusFieldName.right = new FormAttachment(middle, -margin);
- fdlCorpusFieldName.top = new FormAttachment(wSpacer, margin);
- wlCorpusFieldName.setLayoutData(fdlCorpusFieldName);
-
- wCorpusFieldName = new CCombo(shell, BORDER | READ_ONLY);
- PropsUi.setLook(wCorpusFieldName);
- wCorpusFieldName.addListener(SWT.Modify, lsMod);
- FormData fdCorpusFieldName = new FormData();
- fdCorpusFieldName.left = new FormAttachment(middle, 0);
- fdCorpusFieldName.top = new FormAttachment(wSpacer, margin);
- fdCorpusFieldName.right = new FormAttachment(100, 0);
- wCorpusFieldName.setLayoutData(fdCorpusFieldName);
- wCorpusFieldName.addFocusListener(
- new FocusListener() {
- @Override
- public void focusLost(FocusEvent e) {}
-
+ widgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ shell,
+ variables,
+ wTransformName,
+ wOk,
+ DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ input);
+ widgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
@Override
- public void focusGained(FocusEvent e) {
- Cursor busy = new Cursor(shell.getDisplay(), CURSOR_WAIT);
- shell.setCursor(busy);
- get();
- shell.setCursor(null);
- busy.dispose();
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget,
String widgetId) {
+ input.setChanged();
}
});
- // parallelism
- Label wlParallelism = new Label(shell, RIGHT);
- wlParallelism.setText(getString(PKG,
"DetectLanguageDialog.Parallelism.Label"));
- PropsUi.setLook(wlParallelism);
- FormData fdlParallelism = new FormData();
- fdlParallelism.left = new FormAttachment(0, 0);
- fdlParallelism.top = new FormAttachment(wCorpusFieldName, margin);
- fdlParallelism.right = new FormAttachment(middle, -margin);
- wlParallelism.setLayoutData(fdlParallelism);
-
- wParallelism = new Button(shell, CHECK);
- wParallelism.setSelection(input.isParallelism());
- PropsUi.setLook(wParallelism);
- wParallelism.setToolTipText(getString(PKG,
"DetectLanguageDialog.Parallelism.Tooltip"));
- FormData fdParallelism = new FormData();
- fdParallelism.left = new FormAttachment(middle, 0);
- fdParallelism.top = new FormAttachment(wCorpusFieldName, margin);
- fdParallelism.right = new FormAttachment(100, 0);
- wParallelism.setLayoutData(fdParallelism);
- wParallelism.addListener(SWT.Selection, lsMod);
-
- getData();
+ setFieldComboValues();
+ // Filling the field combo is not a change
input.setChanged(changed);
+
focusTransformName();
BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
-
return transformName;
}
- /** Copy information from the meta-data input to the dialog fields. */
- public void getData() {
-
- if (input.getCorpusField() != null) {
- wCorpusFieldName.setText(input.getCorpusField());
+ /**
+ * Stream field names cannot come from {@code comboValuesMethod}, which is
handed only a log
+ * channel and a metadata provider, so the field combo is filled once the
widgets exist. The
+ * configured field stays when the previous fields cannot be read.
+ */
+ private void setFieldComboValues() {
+ try {
+ IRowMeta fields = pipelineMeta.getPrevTransformFields(variables,
transformName);
+ String[] names = fields == null ? new String[0] : fields.getFieldNames();
+ widgets.setComboValues(DetectLanguageMeta.WIDGET_CORPUS_FIELD, names);
+ // Setting items clears the widget's text, so put the configured field
back afterwards.
+ if (!Utils.isEmpty(input.getCorpusField())) {
+ setComboText(DetectLanguageMeta.WIDGET_CORPUS_FIELD,
input.getCorpusField());
+ }
+ } catch (Exception e) {
+ LogChannel.UI.logError(
+ BaseMessages.getString(PKG,
"DetectLanguageDialog.FailedToGetFields.DialogMessage"), e);
}
+ }
- wParallelism.setSelection(input.isParallelism());
+ private void setComboText(String widgetId, String text) {
+ Control control = widgets.getWidgetsMap().get(widgetId);
+ if (control == null || control.isDisposed()) {
+ return;
+ }
+ if (control instanceof ComboVar comboVar) {
+ comboVar.setText(text);
+ } else if (control instanceof Combo combo) {
+ combo.setText(text);
+ }
}
private void cancel() {
@@ -154,32 +118,12 @@ public class DetectLanguageDialog extends
BaseTransformDialog implements ITransf
}
private void ok() {
- if (isEmpty(wTransformName.getText())) {
+ if (Utils.isEmpty(wTransformName.getText())) {
return;
}
- input.setCorpusField(wCorpusFieldName.getText());
- input.setParallelism(wParallelism.getSelection());
-
- transformName = wTransformName.getText(); // return value
-
+ widgets.getWidgetsContents(input,
DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID);
+ transformName = wTransformName.getText();
+ input.setChanged();
dispose();
}
-
- private void get() {
- if (!gotPreviousFields) {
- try {
- IRowMeta r = pipelineMeta.getPrevTransformFields(variables,
transformName);
- if (r != null) {
- ComboItems.setItemsKeepingText(wCorpusFieldName, r.getFieldNames());
- }
- gotPreviousFields = true;
- } catch (HopException ke) {
- new ErrorDialog(
- shell,
- getString(PKG,
"DetectLanguageDialog.FailedToGetFields.DialogTitle"),
- getString(PKG,
"DetectLanguageDialog.FailedToGetFields.DialogMessage"),
- ke);
- }
- }
- }
}
diff --git
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
index efb62eddbc..0f65485c4e 100644
---
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
+++
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
@@ -14,21 +14,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hop.pipeline.transforms.language;
import static org.apache.hop.core.ICheckResult.TYPE_RESULT_ERROR;
import static org.apache.hop.core.ICheckResult.TYPE_RESULT_OK;
+import static org.apache.hop.core.ICheckResult.TYPE_RESULT_WARNING;
import static org.apache.hop.core.util.Utils.isEmpty;
+import com.github.pemistahl.lingua.api.Language;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Locale;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaNumber;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -37,6 +50,9 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
+@Getter
+@Setter
+@GuiPlugin
@Transform(
id = "DetectLanguage",
image = "detectlanguage.svg",
@@ -48,18 +64,88 @@ import org.apache.hop.pipeline.transform.TransformMeta;
public class DetectLanguageMeta extends BaseTransformMeta<DetectLanguage,
DetectLanguageData> {
private static final Class<?> PKG = DetectLanguageMeta.class;
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
"DetectLanguage.Dialog";
+ public static final String WIDGET_CORPUS_FIELD = "corpusField";
+ public static final String WIDGET_PARALLELISM = "parallelism";
+ public static final String WIDGET_LOW_ACCURACY = "lowAccuracy";
+ public static final String WIDGET_LANGUAGES = "languages";
+
+ private static final String GROUP_OPTIONS =
"i18n::DetectLanguage.Group.Options";
+ private static final String GROUP_LANGUAGES =
"i18n::DetectLanguage.Group.Languages";
+
+ @GuiWidgetElement(
+ id = WIDGET_CORPUS_FIELD,
+ order = "0100",
+ type = GuiElementType.COMBO,
+ label = "i18n::DetectLanguageDialog.CorpusFieldName.Label",
+ toolTip = "i18n::DetectLanguageDialog.CorpusFieldName.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_OPTIONS,
+ groupOrder = "0100")
@HopMetadataProperty(
key = "corpusField",
injectionKey = "FIELD",
injectionKeyDescription = "DetectLanguage.Injection.CorpusField")
private String corpusField;
+ @GuiWidgetElement(
+ id = WIDGET_PARALLELISM,
+ order = "0200",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::DetectLanguageDialog.Parallelism.Label",
+ toolTip = "i18n::DetectLanguageDialog.Parallelism.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_OPTIONS,
+ groupOrder = "0100")
@HopMetadataProperty(
key = "parallelism",
injectionKey = "PARALLISM",
injectionKeyDescription = "DetectLanguage.Injection.Parallelism")
private boolean parallelism = false;
+ /**
+ * Lingua's low accuracy mode: only the trigram models are used. Faster and
much lighter on
+ * memory, less accurate on short text. Off for pipelines saved before the
option existed.
+ */
+ @GuiWidgetElement(
+ id = WIDGET_LOW_ACCURACY,
+ order = "0300",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::DetectLanguageDialog.LowAccuracy.Label",
+ toolTip = "i18n::DetectLanguageDialog.LowAccuracy.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_OPTIONS,
+ groupOrder = "0100")
+ @HopMetadataProperty(
+ key = "lowAccuracy",
+ injectionKey = "LOW_ACCURACY",
+ injectionKeyDescription = "DetectLanguage.Injection.LowAccuracy")
+ private boolean lowAccuracy = false;
+
+ /**
+ * The languages the transform may answer. Empty means every spoken language
Lingua knows, which
+ * is what the transform did before the option existed.
+ */
+ @GuiWidgetElement(
+ id = WIDGET_LANGUAGES,
+ order = "0400",
+ type = GuiElementType.TABLE,
+ toolTip = "i18n::DetectLanguageDialog.Languages.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = GROUP_LANGUAGES,
+ groupOrder = "0200",
+ tableRows = 8)
+ @HopMetadataProperty(
+ key = "language",
+ groupKey = "languages",
+ injectionGroupKey = "LANGUAGES",
+ injectionGroupDescription = "DetectLanguage.Injection.Languages")
+ private List<DetectLanguageChoice> languages = new ArrayList<>();
+
public DetectLanguageMeta() {
super();
}
@@ -67,6 +153,77 @@ public class DetectLanguageMeta extends
BaseTransformMeta<DetectLanguage, Detect
@Override
public void setDefault() {
parallelism = false;
+ lowAccuracy = false;
+ languages = new ArrayList<>();
+ }
+
+ /** The names offered in the languages table: every language Lingua can
detect. */
+ public List<String> languageNames(ILogChannel log, IHopMetadataProvider
metadataProvider) {
+ return Language.all().stream().map(Language::name).sorted().toList();
+ }
+
+ /**
+ * Resolves the configured languages.
+ *
+ * @return the languages to detect, without duplicates and in the configured
order; empty when
+ * none is configured, which means all spoken languages
+ * @throws HopException when a name is not a Lingua language or when exactly
one language is
+ * configured: detection needs at least two to choose from
+ */
+ public List<Language> resolveLanguages(IVariables variables) throws
HopException {
+ return resolveLanguages(variables, null);
+ }
+
+ /**
+ * Resolves the configured languages, leaving out the names that still hold
a variable.
+ *
+ * @param unresolved when not null, a name that still holds a variable after
resolving is added
+ * here instead of failing: Verify does not know the values a pipeline
only gets when it runs
+ */
+ List<Language> resolveLanguages(IVariables variables, List<String>
unresolved)
+ throws HopException {
+ List<Language> resolved = new ArrayList<>();
+ if (languages == null) {
+ return resolved;
+ }
+ for (DetectLanguageChoice choice : languages) {
+ String name = choice == null ? null : choice.getName();
+ if (variables != null) {
+ name = variables.resolve(name);
+ }
+ if (StringUtils.isBlank(name)) {
+ continue;
+ }
+ if (unresolved != null && StringUtil.containsVariableToken(name)) {
+ unresolved.add(name.trim());
+ continue;
+ }
+ Language language = findLanguage(name.trim());
+ if (language == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"DetectLanguage.Error.UnknownLanguage", name.trim()));
+ }
+ if (!resolved.contains(language)) {
+ resolved.add(language);
+ }
+ }
+ // With names still to be resolved, the number of languages is not known
yet
+ if (resolved.size() == 1 && (unresolved == null || unresolved.isEmpty())) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "DetectLanguage.Error.SingleLanguage",
resolved.get(0).name()));
+ }
+ return resolved;
+ }
+
+ private static Language findLanguage(String name) {
+ String upper = name.toUpperCase(Locale.ROOT);
+ for (Language language : Language.all()) {
+ if (language.name().equals(upper)) {
+ return language;
+ }
+ }
+ return null;
}
@Override
@@ -77,7 +234,6 @@ public class DetectLanguageMeta extends
BaseTransformMeta<DetectLanguage, Detect
TransformMeta nextTransform,
IVariables variables,
IHopMetadataProvider metadataProvider) {
-
// name = getParentTransformMeta().getName()
valueMetaString(r, name, "detected_language");
valueMetaNumber(r, name, "detected_language_confidence");
@@ -107,7 +263,6 @@ public class DetectLanguageMeta extends
BaseTransformMeta<DetectLanguage, Detect
IVariables variables,
IHopMetadataProvider metadataProvider) {
CheckResult cr;
-
if (isEmpty(corpusField)) {
cr =
new CheckResult(
@@ -123,6 +278,33 @@ public class DetectLanguageMeta extends
BaseTransformMeta<DetectLanguage, Detect
}
remarks.add(cr);
+ try {
+ List<String> unresolved = new ArrayList<>();
+ List<Language> resolved = resolveLanguages(variables, unresolved);
+ if (!unresolved.isEmpty()) {
+ cr =
+ new CheckResult(
+ TYPE_RESULT_WARNING,
+ BaseMessages.getString(
+ PKG,
+
"DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime",
+ String.join(", ", unresolved)),
+ transformMeta);
+ } else {
+ cr =
+ new CheckResult(
+ TYPE_RESULT_OK,
+ resolved.isEmpty()
+ ? BaseMessages.getString(PKG,
"DetectLanguageMeta.CheckResult.AllLanguages")
+ : BaseMessages.getString(
+ PKG, "DetectLanguageMeta.CheckResult.LanguagesOK",
resolved.size()),
+ transformMeta);
+ }
+ } catch (HopException e) {
+ cr = new CheckResult(TYPE_RESULT_ERROR, e.getMessage().trim(),
transformMeta);
+ }
+ remarks.add(cr);
+
// See if we have input streams leading to this transform!
if (input.length > 0) {
cr =
@@ -145,20 +327,4 @@ public class DetectLanguageMeta extends
BaseTransformMeta<DetectLanguage, Detect
public boolean supportsErrorHandling() {
return true;
}
-
- public String getCorpusField() {
- return corpusField;
- }
-
- public void setCorpusField(String corpusField) {
- this.corpusField = corpusField;
- }
-
- public boolean isParallelism() {
- return parallelism;
- }
-
- public void setParallelism(boolean parallelism) {
- this.parallelism = parallelism;
- }
}
diff --git
a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
index be58a587f5..a319f4c8a0 100644
---
a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
+++
b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
@@ -40,3 +40,19 @@
DetectLanguageMeta.CheckResult.ReceivingInfoFromOtherTransforms=Transform is rec
DetectLanguageMeta.CheckResult.ErrorOccurred=An error occurred:
DetectLanguageDialog.Parallelism.Label=Use Parallelism
DetectLanguageDialog.Parallelism.Tooltip=Process the rows in parallel
+DetectLanguageDialog.CorpusFieldName.Tooltip=The field with the text to examine
+DetectLanguageDialog.LowAccuracy.Label=Low accuracy mode
+DetectLanguageDialog.LowAccuracy.Tooltip=Use only the three-letter language
models: faster and much lighter on memory, but less accurate. Meant for longer
texts: short texts of a few words are often given the wrong language.
+DetectLanguageDialog.Languages.Tooltip=The languages the text can be in. Leave
the list empty to detect every spoken language. A short list is faster and uses
less memory; a text in a language that is not listed is given one of the listed
languages or UNKNOWN.
+DetectLanguage.Group.Options=Options
+DetectLanguage.Group.Languages=Languages to detect
+DetectLanguage.Column.Language.Label=Language
+DetectLanguage.Injection.LowAccuracy=Low accuracy mode
+DetectLanguage.Injection.Languages=Languages to detect
+DetectLanguage.Injection.Language=Language name
+DetectLanguage.Error.UnknownLanguage=''{0}'' is not a language this transform
can detect. Use a name from the list, for example ENGLISH.
+DetectLanguage.Error.SingleLanguage=Only one language is listed ({0}). List at
least two languages, or leave the list empty to detect every spoken language.
+DetectLanguage.Error.DetectorNotBuilt=Unable to prepare the language detection
+DetectLanguageMeta.CheckResult.AllLanguages=Every spoken language is detected.
+DetectLanguageMeta.CheckResult.LanguagesOK={0} languages are detected.
+DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime=These languages are
only known when the pipeline runs: {0}
diff --git
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
index 3806cae064..75ca03e304 100644
---
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
+++
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
@@ -23,18 +23,15 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.github.pemistahl.lingua.api.LanguageDetector;
-import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
import java.lang.reflect.Field;
import java.util.List;
import java.util.TreeMap;
@@ -48,12 +45,12 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.mockito.MockedStatic;
@Timeout(10)
class DetectLanguageAsyncTest {
@@ -72,15 +69,14 @@ class DetectLanguageAsyncTest {
PipelineMeta pipelineMeta = new PipelineMeta();
transformMeta = spy(new TransformMeta("test", meta));
pipelineMeta.addTransform(transformMeta);
- try (MockedStatic<LanguageDetectorBuilder> factory =
- mockStatic(LanguageDetectorBuilder.class)) {
- LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class,
RETURNS_SELF);
- LanguageDetector detector = mock(LanguageDetector.class);
-
factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder);
- when(builder.build()).thenReturn(detector);
-
when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new
TreeMap<>());
- transform = spy(new DetectLanguage(transformMeta, meta, data, 0,
pipelineMeta, null));
- }
+ LanguageDetector detector = mock(LanguageDetector.class);
+ when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new
TreeMap<>());
+ data.detector = detector;
+ data.corpusField = "input";
+ transform = spy(new DetectLanguage(transformMeta, meta, data, 0,
pipelineMeta, null));
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("input"));
+ transform.setInputRowMeta(inputRowMeta);
transform.first = false;
data.indexOfCorpusField = 0;
data.indexOfDetectedLanguage = 1;
diff --git
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
index e4cda78ed1..c13196035c 100644
---
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
+++
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
@@ -14,15 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hop.pipeline.transforms.language;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.github.pemistahl.lingua.api.Language;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
+import org.w3c.dom.Node;
class DetectLanguageMetaTest {
@RegisterExtension
@@ -30,11 +43,170 @@ class DetectLanguageMetaTest {
@Test
void testLoadSave() throws HopException {
- List<String> attributes = List.of("corpusField", "parallelism");
+ List<String> attributes = List.of("corpusField", "parallelism",
"lowAccuracy");
+ // the languages list has its own round trip test below
LoadSaveTester<DetectLanguageMeta> loadSaveTester =
- new LoadSaveTester<>(DetectLanguageMeta.class, attributes);
+ new LoadSaveTester<>(
+ DetectLanguageMeta.class,
+ attributes,
+ List.of("languages"),
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ null);
loadSaveTester.testSerialization();
}
+
+ /** A pipeline saved before the language list and the accuracy mode existed.
*/
+ @Test
+ void aTransformSavedBeforeTheNewOptionsKeepsItsBehavior() throws Exception {
+ DetectLanguageMeta meta =
+ load(
+
"<transform><corpusField>comment</corpusField><parallelism>Y</parallelism></transform>");
+
+ assertEquals("comment", meta.getCorpusField());
+ assertTrue(meta.isParallelism());
+ assertFalse(meta.isLowAccuracy());
+ assertTrue(meta.getLanguages().isEmpty());
+ // no languages listed: every spoken language, as before
+ assertTrue(meta.resolveLanguages(new Variables()).isEmpty());
+ }
+
+ @Test
+ void theNewOptionsSurviveAnXmlRoundTrip() throws Exception {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.setLowAccuracy(true);
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("FRENCH"));
+
+ DetectLanguageMeta copy = load("<transform>" + meta.getXml() +
"</transform>");
+
+ assertEquals("comment", copy.getCorpusField());
+ assertFalse(copy.isParallelism());
+ assertTrue(copy.isLowAccuracy());
+ assertEquals(
+ List.of("ENGLISH", "FRENCH"),
+
copy.getLanguages().stream().map(DetectLanguageChoice::getName).toList());
+ }
+
+ @Test
+ void languagesResolveWithVariablesAnyCaseAndNoDuplicates() throws Exception {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("english"));
+ meta.getLanguages().add(new DetectLanguageChoice(" ${SECOND} "));
+ meta.getLanguages().add(new DetectLanguageChoice(""));
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ Variables variables = new Variables();
+ variables.setVariable("SECOND", "French");
+
+ assertEquals(List.of(Language.ENGLISH, Language.FRENCH),
meta.resolveLanguages(variables));
+ }
+
+ @Test
+ void anUnknownLanguageIsRefused() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+
+ HopException e = assertThrows(HopException.class, () ->
meta.resolveLanguages(new Variables()));
+ assertTrue(e.getMessage().contains("KLINGON"));
+ }
+
+ @Test
+ void aSingleLanguageIsRefused() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+
+ HopException e = assertThrows(HopException.class, () ->
meta.resolveLanguages(new Variables()));
+ assertTrue(e.getMessage().contains("ENGLISH"));
+ }
+
+ @Test
+ void theLanguageListOffersEveryLinguaLanguage() {
+ List<String> names = new DetectLanguageMeta().languageNames(null, null);
+
+ assertEquals(Language.all().size(), names.size());
+ assertTrue(names.contains("ENGLISH"));
+ assertFalse(names.contains("UNKNOWN"));
+ }
+
+ @Test
+ void checkReportsABadLanguageList() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("Detect");
+ meta.check(
+ remarks,
+ null,
+ transformMeta,
+ null,
+ new String[] {"in"},
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && r.getText().contains("KLINGON")));
+ }
+
+ /** A language from a variable that only gets a value when the pipeline runs
is not an error. */
+ @Test
+ void checkLeavesLanguagesFromRuntimeVariablesToTheRun() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}"));
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("Detect");
+ meta.check(
+ remarks,
+ null,
+ transformMeta,
+ null,
+ new String[] {"in"},
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertTrue(remarks.stream().noneMatch(r -> r.getType() ==
ICheckResult.TYPE_RESULT_ERROR));
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("${SECOND_LANGUAGE}")));
+ }
+
+ @Test
+ void runningWithAnUnresolvedLanguageStillFails() {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}"));
+
+ assertThrows(HopException.class, () -> meta.resolveLanguages(new
Variables()));
+ }
+
+ private static DetectLanguageMeta load(String xml) throws Exception {
+ Node node = XmlHandler.getSubNode(XmlHandler.loadXmlString(xml),
"transform");
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.loadXml(node, new MemoryMetadataProvider());
+ return meta;
+ }
}
diff --git
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
new file mode 100644
index 0000000000..6f936af4e2
--- /dev/null
+++
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.language;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import com.github.pemistahl.lingua.api.Language;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class DetectLanguageTest {
+
+ private static final String ENGLISH_TEXT =
+ "The weather is lovely today and we are going for a long walk in the
park";
+ private static final String FRENCH_TEXT =
+ "Il fait très beau aujourd'hui et nous allons nous promener dans le
parc";
+ private static final String GERMAN_TEXT =
+ "Das Wetter ist heute sehr schön und wir gehen im Park spazieren";
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper<DetectLanguageMeta, DetectLanguageData>
mockHelper;
+
+ /** Disposed after every test, also a failed one: the Lingua models live in
static maps. */
+ private final List<TestDetectLanguage> transforms = new ArrayList<>();
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>(
+ "Detect language", DetectLanguageMeta.class,
DetectLanguageData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ when(mockHelper.transformMeta.getCopies(any())).thenReturn(1);
+ when(mockHelper.transformMeta.isPartitioned()).thenReturn(false);
+ when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(false);
+ }
+
+ @AfterEach
+ void tearDown() {
+ for (TestDetectLanguage transform : transforms) {
+ transform.dispose();
+ }
+ mockHelper.cleanUp();
+ }
+
+ /** Without options the detector is what the transform always built. */
+ @Test
+ void theDefaultDetectorCoversEverySpokenLanguageWithHighAccuracy() {
+ LanguageDetectorBuilder builder = DetectLanguage.createBuilder(List.of(),
false);
+
+ assertEquals(Language.allSpokenOnes(), builder.getLanguages$lingua());
+ assertFalse(builder.isLowAccuracyModeEnabled$lingua());
+ assertTrue(builder.isEveryLanguageModelPreloaded$lingua());
+ }
+
+ @Test
+ void theConfiguredLanguagesAndAccuracyModeReachTheDetector() {
+ LanguageDetectorBuilder builder =
+ DetectLanguage.createBuilder(List.of(Language.ENGLISH,
Language.FRENCH), true);
+
+ assertEquals(List.of(Language.ENGLISH, Language.FRENCH),
builder.getLanguages$lingua());
+ assertTrue(builder.isLowAccuracyModeEnabled$lingua());
+ }
+
+ @Test
+ void rowsGetTheirLanguageAndBlankTextsPassAsUnknown() throws Exception {
+ DetectLanguage transform = createTransform(meta(false, false));
+ assertTrue(transform.init());
+
+ BlockingRowSet output =
+ run(transform, row(1L, ENGLISH_TEXT), row(2L, null), row(3L,
FRENCH_TEXT), row(4L, " "));
+
+ Object[] english = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(1L, english[0]);
+ assertEquals("ENGLISH", english[2]);
+ assertNotNull(english[3]);
+
+ // a null or blank text used to drop the row
+ Object[] empty = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(2L, empty[0]);
+ assertEquals("UNKNOWN", empty[2]);
+ assertNull(empty[3]);
+
+ assertEquals("FRENCH", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+
+ Object[] blank = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(4L, blank[0]);
+ assertEquals("UNKNOWN", blank[2]);
+ assertNull(blank[3]);
+
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ }
+
+ @Test
+ void theCorpusFieldCanComeFromAVariable() throws Exception {
+ DetectLanguageMeta meta = meta(false, false);
+ meta.setCorpusField("${TEXT_FIELD}");
+ DetectLanguage transform = createTransform(meta);
+ assertTrue(transform.init());
+ transform.setVariable("TEXT_FIELD", "comment");
+
+ BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT));
+
+ assertEquals("GERMAN", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+ }
+
+ @Test
+ void lowAccuracyModeFillsBothFields() throws Exception {
+ DetectLanguage transform = createTransform(meta(false, true));
+ assertTrue(transform.init());
+
+ BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT));
+
+ Object[] german = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals("GERMAN", german[2]);
+ assertNotNull(german[3]);
+ }
+
+ @Test
+ void parallelModeWritesEveryRow() throws Exception {
+ DetectLanguage transform = createTransform(meta(true, false));
+ assertTrue(transform.init());
+
+ BlockingRowSet output =
+ run(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L,
GERMAN_TEXT));
+
+ Map<Object, Object> languageById = new HashMap<>();
+ for (int i = 0; i < 3; i++) {
+ Object[] row = output.getRowWait(5, TimeUnit.SECONDS);
+ assertNotNull(row);
+ languageById.put(row[0], row[2]);
+ }
+ assertEquals(Map.of(1L, "ENGLISH", 2L, "FRENCH", 3L, "GERMAN"),
languageById);
+ }
+
+ /** A failing parallel job used to leave the transform waiting forever at
the end. */
+ @Test
+ void aFailingParallelJobStopsTheTransform() {
+ DetectLanguage transform = createTransform(meta(true, false),
DetectLanguageTest::noRoom);
+ assertTrue(transform.init());
+
+ HopException e =
+ assertTimeoutPreemptively(
+ Duration.ofSeconds(30),
+ () -> assertThrows(HopException.class, () -> run(transform,
row(1L, ENGLISH_TEXT))));
+ assertTrue(e.getMessage().contains("no room for this row"));
+ }
+
+ @Test
+ void aFailingParallelRowGoesToTheErrorHop() throws Exception {
+ when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true);
+ TestDetectLanguage transform = createTransform(meta(true, false),
DetectLanguageTest::noRoom);
+ assertTrue(transform.init());
+
+ assertTimeoutPreemptively(
+ Duration.ofSeconds(30), () -> run(transform, row(1L, ENGLISH_TEXT),
row(2L, FRENCH_TEXT)));
+
+ List<ErrorRow> errorRows = new ArrayList<>(transform.errorRows);
+ assertEquals(2, errorRows.size());
+ for (ErrorRow errorRow : errorRows) {
+ assertEquals("comment", errorRow.fieldNames());
+ assertEquals("DetectLanguage001", errorRow.errorCodes());
+ }
+ }
+
+ /** An Error says nothing about the row: it stops the transform, also with
error handling on. */
+ @Test
+ void anErrorInAParallelJobStopsTheTransform() {
+ when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true);
+ TestDetectLanguage transform =
+ createTransform(
+ meta(true, false),
+ (rowMeta, row) -> {
+ throw new NoClassDefFoundError("simulated failure");
+ });
+ assertTrue(transform.init());
+
+ HopException e =
+ assertTimeoutPreemptively(
+ Duration.ofSeconds(30),
+ () -> assertThrows(HopException.class, () -> run(transform,
row(1L, ENGLISH_TEXT))));
+ assertTrue(e.getMessage().contains("simulated failure"));
+ assertTrue(transform.errorRows.isEmpty());
+ }
+
+ /**
+ * After an error or a stop the transform is disposed without reaching the
end of its input.
+ * Parallel jobs that still run use the detector, so dispose waits for them.
+ */
+ @Test
+ void disposeWaitsForTheParallelJobsStillRunning() throws Exception {
+ CountDownLatch writing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ TestDetectLanguage transform =
+ createTransform(
+ meta(true, false),
+ (rowMeta, row) -> {
+ writing.countDown();
+ awaitQuietly(release);
+ });
+ assertTrue(transform.init());
+
+ connect(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L,
GERMAN_TEXT));
+ for (int i = 0; i < 3; i++) {
+ assertTrue(transform.processRow());
+ }
+ assertTrue(writing.await(30, TimeUnit.SECONDS));
+
+ // What RunThread does when processRow fails or the pipeline is stopped
+ transform.stopAll();
+ Thread disposer = new Thread(transform::dispose, "dispose");
+ disposer.start();
+ disposer.join(300);
+ assertTrue(disposer.isAlive(), "dispose returned while a job was still
running");
+ assertNotNull(transform.getData().detector);
+
+ release.countDown();
+ disposer.join(30_000);
+ assertFalse(disposer.isAlive());
+ assertNull(transform.getData().detector);
+ assertTrue(transform.errorRows.isEmpty());
+ }
+
+ @Test
+ void anUnknownLanguageFailsAtStart() {
+ DetectLanguageMeta meta = meta(false, false);
+ meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+
+ assertFalse(createTransform(meta).init());
+ }
+
+ private static DetectLanguageMeta meta(boolean parallelism, boolean
lowAccuracy) {
+ DetectLanguageMeta meta = new DetectLanguageMeta();
+ meta.setCorpusField("comment");
+ meta.setParallelism(parallelism);
+ meta.setLowAccuracy(lowAccuracy);
+ // three languages keep the models small; the default is covered by the
builder test
+ meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+ meta.getLanguages().add(new DetectLanguageChoice("FRENCH"));
+ meta.getLanguages().add(new DetectLanguageChoice("GERMAN"));
+ return meta;
+ }
+
+ private static Object[] row(Long id, String comment) {
+ return new Object[] {id, comment};
+ }
+
+ private static void noRoom(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ throw new HopTransformException("no room for this row");
+ }
+
+ private static void awaitQuietly(CountDownLatch latch) {
+ try {
+ latch.await(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /** Connects an input holding the rows, and an output, to the transform. */
+ private static BlockingRowSet connect(DetectLanguage transform, Object[]...
rows) {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("comment"));
+
+ BlockingRowSet input = new BlockingRowSet(100);
+ input.setThreadNameFromToCopy("previous", 0, "Detect language", 0);
+ input.setRowMeta(rowMeta);
+ for (Object[] row : rows) {
+ input.putRow(rowMeta, row);
+ }
+ input.setDone();
+
+ BlockingRowSet output = new BlockingRowSet(100);
+ output.setThreadNameFromToCopy("Detect language", 0, "next", 0);
+
+ transform.addRowSetToInputRowSets(input);
+ transform.addRowSetToOutputRowSets(output);
+ return output;
+ }
+
+ /** Feeds the rows through the transform until it reports the end of its
input. */
+ private static BlockingRowSet run(DetectLanguage transform, Object[]... rows)
+ throws HopException {
+ BlockingRowSet output = connect(transform, rows);
+ List<Boolean> results = new ArrayList<>();
+ boolean more = true;
+ while (more) {
+ more = transform.processRow();
+ results.add(more);
+ }
+ assertEquals(rows.length + 1, results.size());
+ return output;
+ }
+
+ private TestDetectLanguage createTransform(DetectLanguageMeta meta) {
+ return createTransform(meta, null);
+ }
+
+ private TestDetectLanguage createTransform(DetectLanguageMeta meta, RowWrite
write) {
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ TestDetectLanguage transform = new TestDetectLanguage(mockHelper, meta,
write);
+ transforms.add(transform);
+ return transform;
+ }
+
+ /** What a test's transform does with an output row instead of writing it. */
+ @FunctionalInterface
+ private interface RowWrite {
+ void write(IRowMeta rowMeta, Object[] row) throws HopTransformException;
+ }
+
+ private record ErrorRow(Object id, String fieldNames, String errorCodes) {}
+
+ /** The transform, with the writes a test controls and the error rows it
sends. */
+ private static class TestDetectLanguage extends DetectLanguage {
+ private final RowWrite write;
+ final List<ErrorRow> errorRows = Collections.synchronizedList(new
ArrayList<>());
+
+ TestDetectLanguage(
+ TransformMockHelper<DetectLanguageMeta, DetectLanguageData> helper,
+ DetectLanguageMeta meta,
+ RowWrite write) {
+ super(
+ helper.transformMeta,
+ meta,
+ new DetectLanguageData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ this.write = write;
+ }
+
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ if (write == null) {
+ super.putRow(rowMeta, row);
+ } else {
+ write.write(rowMeta, row);
+ }
+ }
+
+ @Override
+ public void putError(
+ IRowMeta rowMeta,
+ Object[] row,
+ long nrErrors,
+ String errorDescriptions,
+ String fieldNames,
+ String errorCodes) {
+ errorRows.add(new ErrorRow(row[0], fieldNames, errorCodes));
+ }
+ }
+}