This is an automated email from the ASF dual-hosted git repository.

hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 0f809ad9b0 Detect language: choose the languages to detect and a 
faster low accu… (#9434)
0f809ad9b0 is described below

commit 0f809ad9b05210ddaab19080de1846948d34b4c6
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Nov 16 17:26:45 2026 +0100

    Detect language: choose the languages to detect and a faster low accu… 
(#9434)
    
    * Detect language: choose the languages to detect and a faster low accuracy 
mode, fixes #9433
    
    * Detect language: integration test for the language choice, low accuracy 
mode and blank texts, #9433
    
    Co-Authored-By: Claude Opus 5.5 <[email protected]>
    
    ---------
    
    Co-authored-by: Claude Opus 5.5 <[email protected]>
---
 .../pages/pipeline/transforms/detectlanguage.adoc  |  31 ++
 .../transforms/0128-detect-language.hpl            | 217 +++++++++++
 .../golden-0128-detect-language-chosen.csv         |   6 +
 .../datasets/golden-0128-detect-language.csv       |   6 +
 .../transforms/main-0128-detect-language.hwf       |  79 ++++
 .../golden-0128-detect-language-chosen.json        |  24 ++
 .../dataset/golden-0128-detect-language.json       |  24 ++
 .../unit-test/0128-detect-language UNIT.json       |  49 +++
 .../transforms/language/DetectLanguage.java        | 115 ++++--
 .../transforms/language/DetectLanguageChoice.java  |  50 +++
 .../transforms/language/DetectLanguageData.java    |   7 +
 .../transforms/language/DetectLanguageDialog.java  | 170 +++------
 .../transforms/language/DetectLanguageMeta.java    | 204 ++++++++++-
 .../language/messages/messages_en_US.properties    |  16 +
 .../language/DetectLanguageAsyncTest.java          |  22 +-
 .../language/DetectLanguageMetaTest.java           | 178 ++++++++-
 .../transforms/language/DetectLanguageTest.java    | 401 +++++++++++++++++++++
 17 files changed, 1429 insertions(+), 170 deletions(-)

diff --git 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
index ace4d07ded..6268a61e9f 100644
--- 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
+++ 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/detectlanguage.adoc
@@ -35,8 +35,39 @@ 
https://github.com/pemistahl/lingua?tab=readme-ov-file#3-which-languages-are-sup
 |Option|Description
 |Transform name|Name of the transform.
 This name must be unique throughout the pipeline.
+|Corpus text field|The field with the text to examine.
+A variable is allowed.
+|Use Parallelism|Detect the language of several rows at the same time.
+The rows can leave the transform in another order than they came in.
+|Low accuracy mode|Use only the three-letter language models.
+This is faster and uses far less memory, but it is less accurate: short texts 
of a few words are often given the wrong language.
+Use it for longer texts.
+|Languages to detect|The languages the text can be in, by their Lingua name 
(`ENGLISH`, `FRENCH`, ...).
+A variable is allowed.
+Leave the list empty to detect every spoken language.
+List at least two languages.
 |===
 
+== Output fields
+
+[options="header"]
+|===
+|Field|Description
+|detected_language|The name of the detected language, for example `ENGLISH`, 
or `UNKNOWN`.
+|detected_language_confidence|Lingua's score for that language, relative to 
the other languages.
+The detected language is the one with the highest score, so the value is 
always 1: it is not a probability and does not tell how sure the detection is.
+Empty when the language is `UNKNOWN`.
+|===
+
+A row with an empty text passes with `UNKNOWN` and no confidence.
+The language is also `UNKNOWN` when the two most likely languages score 
exactly the same.
+
+== Performance
+
+Detecting every spoken language loads the models of all 74 spoken languages: 
about 1.4 GB of memory, released when the transform finishes.
+When you know which languages the data can contain, list them: with five 
languages the transform is about 2.5 times as fast and needs about 175 MB.
+A text in a language that is not listed is given one of the listed languages, 
or `UNKNOWN`.
+
 == Installation
 
 This transform is not part of the default Hop client.
diff --git a/integration-tests/transforms/0128-detect-language.hpl 
b/integration-tests/transforms/0128-detect-language.hpl
new file mode 100644
index 0000000000..23f4337844
--- /dev/null
+++ b/integration-tests/transforms/0128-detect-language.hpl
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+  <info>
+    <name>0128-detect-language</name>
+    <name_sync_with_filename>Y</name_sync_with_filename>
+    <description>Detect language with the default options (every spoken 
language, high accuracy, one thread) and with a chosen set of languages, low 
accuracy mode and parallel jobs. Blank and null texts pass with 
UNKNOWN.</description>
+    <extended_description/>
+    <pipeline_version/>
+    <pipeline_type>Normal</pipeline_type>
+    <parameters>
+      <parameter>
+        <name>DETECT_LANGUAGE_FIELD</name>
+        <default_value>text</default_value>
+        <description>The corpus field accepts a variable</description>
+      </parameter>
+      <parameter>
+        <name>DETECT_LANGUAGE_THIRD</name>
+        <default_value>GERMAN</default_value>
+        <description>A language to choose accepts a variable</description>
+      </parameter>
+    </parameters>
+    <capture_transform_performance>N</capture_transform_performance>
+    
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+    
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+    <created_user>-</created_user>
+    <created_date>2026/11/16 12:00:00.000</created_date>
+    <modified_user>-</modified_user>
+    <modified_date>2026/11/16 12:00:00.000</modified_date>
+  </info>
+  <notepads>
+  </notepads>
+  <order>
+    <hop>
+      <from>texts</from>
+      <to>detect, default options</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>texts</from>
+      <to>detect, chosen languages</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>detect, default options</from>
+      <to>validate</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>detect, chosen languages</from>
+      <to>validate chosen</to>
+      <enabled>Y</enabled>
+    </hop>
+  </order>
+  <transform>
+    <name>texts</name>
+    <type>DataGrid</type>
+    <description>English, French and German text, only spaces, and no text at 
all (null).</description>
+    <distribute>N</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <data>
+      <line>
+        <item>1</item>
+        <item>The children are playing in the garden while their parents 
prepare dinner in the kitchen.</item>
+      </line>
+      <line>
+        <item>2</item>
+        <item>Les enfants jouent dans le jardin pendant que leurs parents 
préparent le dîner dans la cuisine.</item>
+      </line>
+      <line>
+        <item>3</item>
+        <item>Die Kinder spielen im Garten, während ihre Eltern in der Küche 
das Abendessen vorbereiten.</item>
+      </line>
+      <line>
+        <item>4</item>
+        <item>   </item>
+      </line>
+      <line>
+        <item>5</item>
+        <item/>
+      </line>
+    </data>
+    <fields>
+      <field>
+        <length>-1</length>
+        <precision>-1</precision>
+        <set_empty_string>N</set_empty_string>
+        <name>id</name>
+        <type>String</type>
+      </field>
+      <field>
+        <length>-1</length>
+        <precision>-1</precision>
+        <set_empty_string>N</set_empty_string>
+        <name>text</name>
+        <type>String</type>
+      </field>
+    </fields>
+    <attributes/>
+    <GUI>
+      <xloc>112</xloc>
+      <yloc>160</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>detect, default options</name>
+    <type>DetectLanguage</type>
+    <description>Every spoken language, high accuracy, one 
thread.</description>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <corpusField>text</corpusField>
+    <parallelism>N</parallelism>
+    <lowAccuracy>N</lowAccuracy>
+    <languages>
+    </languages>
+    <attributes/>
+    <GUI>
+      <xloc>336</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>detect, chosen languages</name>
+    <type>DetectLanguage</type>
+    <description>English, French and German only, low accuracy mode, parallel 
jobs: the rows can come out in another order.</description>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <corpusField>${DETECT_LANGUAGE_FIELD}</corpusField>
+    <parallelism>Y</parallelism>
+    <lowAccuracy>Y</lowAccuracy>
+    <languages>
+      <language>
+        <name>ENGLISH</name>
+      </language>
+      <language>
+        <name>FRENCH</name>
+      </language>
+      <language>
+        <name>${DETECT_LANGUAGE_THIRD}</name>
+      </language>
+    </languages>
+    <attributes/>
+    <GUI>
+      <xloc>336</xloc>
+      <yloc>224</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>validate</name>
+    <type>Dummy</type>
+    <description></description>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+    <GUI>
+      <xloc>560</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>validate chosen</name>
+    <type>Dummy</type>
+    <description></description>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+    <GUI>
+      <xloc>560</xloc>
+      <yloc>224</yloc>
+    </GUI>
+  </transform>
+  <transform_error_handling>
+  </transform_error_handling>
+  <attributes/>
+</pipeline>
diff --git 
a/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv 
b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
new file mode 100644
index 0000000000..782fe6e881
--- /dev/null
+++ 
b/integration-tests/transforms/datasets/golden-0128-detect-language-chosen.csv
@@ -0,0 +1,6 @@
+id,detected_language
+1,ENGLISH
+2,FRENCH
+3,GERMAN
+4,UNKNOWN
+5,UNKNOWN
diff --git 
a/integration-tests/transforms/datasets/golden-0128-detect-language.csv 
b/integration-tests/transforms/datasets/golden-0128-detect-language.csv
new file mode 100644
index 0000000000..782fe6e881
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0128-detect-language.csv
@@ -0,0 +1,6 @@
+id,detected_language
+1,ENGLISH
+2,FRENCH
+3,GERMAN
+4,UNKNOWN
+5,UNKNOWN
diff --git a/integration-tests/transforms/main-0128-detect-language.hwf 
b/integration-tests/transforms/main-0128-detect-language.hwf
new file mode 100644
index 0000000000..9d1831237c
--- /dev/null
+++ b/integration-tests/transforms/main-0128-detect-language.hwf
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+  <name>main-0128-detect-language</name>
+  <name_sync_with_filename>Y</name_sync_with_filename>
+  <description/>
+  <extended_description/>
+  <workflow_version/>
+  <created_user>-</created_user>
+  <created_date>2026/11/16 12:00:00.000</created_date>
+  <modified_user>-</modified_user>
+  <modified_date>2026/11/16 12:00:00.000</modified_date>
+  <parameters>
+    </parameters>
+  <actions>
+    <action>
+      <name>Start</name>
+      <description/>
+      <type>SPECIAL</type>
+      <attributes/>
+      <DayOfMonth>1</DayOfMonth>
+      <hour>12</hour>
+      <intervalMinutes>60</intervalMinutes>
+      <intervalSeconds>0</intervalSeconds>
+      <minutes>0</minutes>
+      <repeat>N</repeat>
+      <schedulerType>0</schedulerType>
+      <weekDay>1</weekDay>
+      <parallel>N</parallel>
+      <xloc>112</xloc>
+      <yloc>64</yloc>
+      <attributes_hac/>
+    </action>
+    <action>
+      <name>0128-detect-language UNIT</name>
+      <description/>
+      <type>RunPipelineTests</type>
+      <attributes/>
+      <test_names>
+        <test_name>
+          <name>0128-detect-language UNIT</name>
+        </test_name>
+      </test_names>
+      <parallel>N</parallel>
+      <xloc>304</xloc>
+      <yloc>64</yloc>
+      <attributes_hac/>
+    </action>
+  </actions>
+  <hops>
+    <hop>
+      <from>Start</from>
+      <to>0128-detect-language UNIT</to>
+      <enabled>Y</enabled>
+      <evaluation>Y</evaluation>
+      <unconditional>Y</unconditional>
+    </hop>
+  </hops>
+  <notepads>
+  </notepads>
+  <attributes/>
+</workflow>
diff --git 
a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
 
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
new file mode 100644
index 0000000000..d9c08c0784
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language-chosen.json
@@ -0,0 +1,24 @@
+{
+  "base_filename": "golden-0128-detect-language-chosen.csv",
+  "name": "golden-0128-detect-language-chosen",
+  "description": "",
+  "dataset_fields": [
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_name": "id",
+      "field_format": ""
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_name": "detected_language",
+      "field_format": ""
+    }
+  ],
+  "folder_name": ""
+}
diff --git 
a/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
 
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
new file mode 100644
index 0000000000..7c1acb1722
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/dataset/golden-0128-detect-language.json
@@ -0,0 +1,24 @@
+{
+  "base_filename": "golden-0128-detect-language.csv",
+  "name": "golden-0128-detect-language",
+  "description": "",
+  "dataset_fields": [
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_name": "id",
+      "field_format": ""
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_name": "detected_language",
+      "field_format": ""
+    }
+  ],
+  "folder_name": ""
+}
diff --git 
a/integration-tests/transforms/metadata/unit-test/0128-detect-language 
UNIT.json 
b/integration-tests/transforms/metadata/unit-test/0128-detect-language UNIT.json
new file mode 100644
index 0000000000..99ba566cdc
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0128-detect-language 
UNIT.json    
@@ -0,0 +1,49 @@
+{
+  "database_replacements": [],
+  "autoOpening": true,
+  "description": "",
+  "persist_filename": "",
+  "test_type": "UNIT_TEST",
+  "variableValues": [],
+  "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+  "golden_data_sets": [
+    {
+      "field_mappings": [
+        {
+          "transform_field": "id",
+          "data_set_field": "id"
+        },
+        {
+          "transform_field": "detected_language",
+          "data_set_field": "detected_language"
+        }
+      ],
+      "field_order": [
+        "id"
+      ],
+      "data_set_name": "golden-0128-detect-language",
+      "transform_name": "validate"
+    },
+    {
+      "field_mappings": [
+        {
+          "transform_field": "id",
+          "data_set_field": "id"
+        },
+        {
+          "transform_field": "detected_language",
+          "data_set_field": "detected_language"
+        }
+      ],
+      "field_order": [
+        "id"
+      ],
+      "data_set_name": "golden-0128-detect-language-chosen",
+      "transform_name": "validate chosen"
+    }
+  ],
+  "input_data_sets": [],
+  "name": "0128-detect-language UNIT",
+  "trans_test_tweaks": [],
+  "pipeline_filename": "./0128-detect-language.hpl"
+}
diff --git 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
index bf569b32e7..611d2aac1d 100644
--- 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
+++ 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguage.java
@@ -18,7 +18,6 @@
 package org.apache.hop.pipeline.transforms.language;
 
 import static com.github.pemistahl.lingua.api.Language.UNKNOWN;
-import static 
com.github.pemistahl.lingua.api.LanguageDetectorBuilder.fromAllSpokenLanguages;
 import static java.lang.System.arraycopy;
 import static java.util.concurrent.ForkJoinPool.commonPool;
 import static org.apache.commons.lang3.StringUtils.isBlank;
@@ -27,7 +26,9 @@ import static org.apache.hop.core.util.Utils.isEmpty;
 
 import com.github.pemistahl.lingua.api.Language;
 import com.github.pemistahl.lingua.api.LanguageDetector;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
 import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 import java.util.SortedMap;
 import java.util.concurrent.ExecutorService;
@@ -44,6 +45,7 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
   private static final Class<?> PKG = DetectLanguage.class;
   private static final String 
CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD =
       "DetectLanguage.Exception.CouldnotFindField";
+  private static final String ERROR_CODE = "DetectLanguage001";
 
   public DetectLanguage(
       TransformMeta transformMeta,
@@ -71,15 +73,52 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
 
   private final Object lock = new Object();
 
-  private final LanguageDetector detector =
-      fromAllSpokenLanguages().withPreloadedLanguageModels().build();
+  /** The outcome of one detection: the language name and its confidence, null 
when unknown. */
+  record Detection(String language, Double confidence) {}
 
-  private void awaitJobs() {
-    awaitJobs(DRAIN_TIMEOUT_MILLIS);
+  /**
+   * The Lingua builder for the configured options.
+   *
+   * @param languages the languages to choose from; empty means all spoken 
languages
+   * @param lowAccuracy use only the trigram models
+   */
+  static LanguageDetectorBuilder createBuilder(List<Language> languages, 
boolean lowAccuracy) {
+    LanguageDetectorBuilder builder =
+        languages.isEmpty()
+            ? LanguageDetectorBuilder.fromAllSpokenLanguages()
+            : LanguageDetectorBuilder.fromLanguages(languages.toArray(new 
Language[0]));
+    if (lowAccuracy) {
+      builder = builder.withLowAccuracyMode();
+    }
+    return builder.withPreloadedLanguageModels();
   }
 
-  private void awaitJobs(final long timeoutMillis) {
+  @Override
+  public boolean init() {
+    if (!super.init()) {
+      return false;
+    }
+    try {
+      data.detector = createBuilder(meta.resolveLanguages(this), 
meta.isLowAccuracy()).build();
+      return true;
+    } catch (Exception e) {
+      logError(BaseMessages.getString(PKG, 
"DetectLanguage.Error.DetectorNotBuilt"), e);
+      return false;
+    }
+  }
+
+  /**
+   * Waits for accepted asynchronous work, at most {@link 
#DRAIN_TIMEOUT_MILLIS}.
+   *
+   * @return true when every job finished, false when some were abandoned
+   */
+  private boolean awaitJobs() {
+    return awaitJobs(DRAIN_TIMEOUT_MILLIS);
+  }
+
+  private boolean awaitJobs(final long timeoutMillis) {
     boolean interrupted = false;
+    boolean drained;
     final long deadline = System.currentTimeMillis() + timeoutMillis;
     synchronized (lock) {
       long remaining = timeoutMillis;
@@ -93,7 +132,8 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
         }
         remaining = deadline - System.currentTimeMillis();
       }
-      if (jobs > 0) {
+      drained = jobs == 0;
+      if (!drained) {
         // A worker is stuck and no provider timeout applies. Do not block 
disposal forever: log and
         // abandon the remaining jobs.
         logBasic(
@@ -107,6 +147,7 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
     if (interrupted) {
       Thread.currentThread().interrupt();
     }
+    return drained;
   }
 
   /**
@@ -150,8 +191,16 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
   @Override
   public void dispose() {
     // The common pool is shared. Drain our work, but never shut the pool down.
-    awaitJobs();
-    detector.unloadLanguageModels();
+    // After an error or a stop the transform ends without finish(), and 
parallel jobs can still be
+    // queued. They use the detector: unloading under them would make them 
fail on a missing
+    // detector, or load the models again with nothing left to unload them.
+    boolean drained = awaitJobs();
+    if (drained && data.detector != null) {
+      // Lingua keeps its models in static maps: release what this transform 
asked for. Another
+      // Detect language transform that is still running loads them again on 
demand.
+      data.detector.unloadLanguageModels();
+      data.detector = null;
+    }
     final boolean shouldLog;
     synchronized (lock) {
       // Only log a failure we have not already surfaced through the run loop, 
so it is not logged
@@ -189,7 +238,9 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
       data.outputRowMeta = getInputRowMeta().clone();
       meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, 
metadataProvider);
 
-      if (isEmpty(meta.getCorpusField())) {
+      // The field combo accepts variables
+      data.corpusField = resolve(meta.getCorpusField());
+      if (isEmpty(data.corpusField)) {
         throw new HopException(
             BaseMessages.getString(PKG, 
"DetectLanguage.Error.CorpusFieldMissing"));
       }
@@ -198,7 +249,6 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
     } // End If first
 
     try {
-      process(r);
       if (isRowLevel()) {
         logRowlevel(
             BaseMessages.getString(
@@ -206,6 +256,7 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
                 "DetectLanguage.LineNumber",
                 getLinesRead() + " : " + getInputRowMeta().getString(r)));
       }
+      process(r);
     } catch (Exception e) {
       // An infrastructure/worker failure must not be attributed to the 
current input row.
       checkFailure();
@@ -216,8 +267,10 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
   }
 
   private void process(Object[] inputRow) throws HopException {
-    String corpus = trim((String) inputRow[data.indexOfCorpusField]);
+    String corpus = trim(getInputRowMeta().getString(inputRow, 
data.indexOfCorpusField));
     if (isBlank(corpus)) {
+      // Nothing to detect: the row passes with an unknown language and no 
confidence.
+      putDetection(inputRow, new Detection(UNKNOWN.name(), null));
       return;
     }
 
@@ -230,11 +283,13 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
 
   private void handleRowError(Object[] row, Exception e) throws HopException {
     if (!getTransformMeta().isDoingErrorHandling()) {
-      throw new HopException("Error processing row", e);
+      throw new HopException(
+          BaseMessages.getString(PKG, 
"DetectLanguage.ErrorInTransformRunning") + e.getMessage(),
+          e);
     }
     // Error output has the same serialization requirement as normal output.
     synchronized (this) {
-      putError(getInputRowMeta(), row, 1, e.toString(), meta.getCorpusField(), 
"DetectLanguage001");
+      putError(getInputRowMeta(), row, 1, e.toString(), data.corpusField, 
ERROR_CODE);
     }
   }
 
@@ -257,11 +312,18 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
           () -> {
             try {
               try {
-                processSync(text, inputRow);
+                // Stopped: nobody wants the row anymore, and dispose() waits 
for the queue to
+                // drain.
+                if (!isStopped() || isSafeStopped()) {
+                  processSync(text, inputRow);
+                }
               } catch (Exception e) {
                 handleRowError(inputRow, e);
               }
             } catch (Throwable e) {
+              // Throwable, not Exception: nobody reads the pool's Future, so 
an Error caught
+              // nowhere else would make the row disappear without a trace. An 
Error says nothing
+              // about the row: it always stops the transform.
               synchronized (lock) {
                 if (failure == null) {
                   failure = e;
@@ -286,16 +348,14 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
     }
   }
 
-  @SuppressWarnings(
-      "javabugs:S2259") // the probabilities come from the map's own keys and 
are never null
   private void cacheIndexPositions() throws HopException {
     if (data.indexOfCorpusField < 0) {
-      data.indexOfCorpusField = 
data.previousRowMeta.indexOfValue(meta.getCorpusField());
+      data.indexOfCorpusField = 
data.previousRowMeta.indexOfValue(data.corpusField);
       if (data.indexOfCorpusField < 0) {
         // The field is unreachable!
         throw new HopException(
             BaseMessages.getString(
-                PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD, 
meta.getCorpusField()));
+                PKG, CONST_DETECT_LANGUAGE_EXCEPTION_COULDNOT_FIND_FIELD, 
data.corpusField));
       }
     }
     if (data.indexOfDetectedLanguage < 0) {
@@ -321,9 +381,13 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
     }
   }
 
+  /**
+   * Detects the language of a text: the most likely language, unless the 
second one is as likely or
+   * closer than Lingua's minimum relative distance, in which case the 
language is unknown.
+   */
   @SuppressWarnings(
       "javabugs:S2259") // the probabilities come from the map's own keys and 
are never null
-  private void processSync(String corpus, Object[] inputRow) throws 
HopTransformException {
+  static Detection detect(LanguageDetector detector, String corpus) {
     SortedMap<Language, Double> confidenceValues = 
detector.computeLanguageConfidenceValues(corpus);
     Language detectedLanguage = UNKNOWN;
     Double confidence = null;
@@ -348,11 +412,18 @@ public class DetectLanguage extends 
BaseTransform<DetectLanguageMeta, DetectLang
         }
       }
     }
+    return new Detection(detectedLanguage.name(), confidence);
+  }
+
+  private void processSync(String corpus, Object[] inputRow) throws 
HopTransformException {
+    putDetection(inputRow, detect(data.detector, corpus));
+  }
 
+  private void putDetection(Object[] inputRow, Detection detection) throws 
HopTransformException {
     Object[] outputRow = new Object[inputRow.length + 2 + OVER_ALLOCATE_SIZE];
     arraycopy(inputRow, 0, outputRow, 0, inputRow.length);
-    outputRow[data.indexOfDetectedLanguage] = detectedLanguage.name();
-    outputRow[data.indexOfDetectedLanguageConfidence] = confidence;
+    outputRow[data.indexOfDetectedLanguage] = detection.language();
+    outputRow[data.indexOfDetectedLanguageConfidence] = detection.confidence();
     putRow(data.outputRowMeta, outputRow);
   }
 
diff --git 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
new file mode 100644
index 0000000000..7b48e58e34
--- /dev/null
+++ 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageChoice.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.language;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiTableColumn;
+import org.apache.hop.core.gui.plugin.GuiTableColumnType;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+/** One language the Detect language transform may answer. */
+@Getter
+@Setter
+public class DetectLanguageChoice {
+
+  /** The Lingua language name, for example ENGLISH. A variable is allowed. */
+  @HopMetadataProperty(
+      key = "name",
+      injectionKey = "LANGUAGE",
+      injectionKeyDescription = "DetectLanguage.Injection.Language")
+  @GuiTableColumn(
+      id = "name",
+      order = "10",
+      type = GuiTableColumnType.COMBO,
+      comboValuesMethod = "languageNames",
+      label = "i18n::DetectLanguage.Column.Language.Label",
+      variables = true,
+      width = 250)
+  private String name = "";
+
+  public DetectLanguageChoice() {}
+
+  public DetectLanguageChoice(String name) {
+    this.name = name;
+  }
+}
diff --git 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
index ef5d72bf98..12f16dbfb3 100644
--- 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
+++ 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageData.java
@@ -17,6 +17,7 @@
 
 package org.apache.hop.pipeline.transforms.language;
 
+import com.github.pemistahl.lingua.api.LanguageDetector;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.pipeline.transform.BaseTransformData;
 import org.apache.hop.pipeline.transform.ITransformData;
@@ -30,6 +31,12 @@ public class DetectLanguageData extends BaseTransformData 
implements ITransformD
   public IRowMeta outputRowMeta;
   public int NrPrevFields;
 
+  /** The name of the corpus field, its variables resolved. */
+  public String corpusField;
+
+  /** Built in init() from the configured languages and accuracy mode. */
+  public LanguageDetector detector;
+
   public DetectLanguageData() {
     super();
     indexOfCorpusField = -1;
diff --git 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
index c5a1fcb766..372b679cee 100644
--- 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
+++ 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageDialog.java
@@ -14,48 +14,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hop.pipeline.transforms.language;
 
-import static org.apache.hop.core.util.Utils.isEmpty;
-import static org.apache.hop.i18n.BaseMessages.getString;
-import static org.eclipse.swt.SWT.BORDER;
-import static org.eclipse.swt.SWT.CHECK;
-import static org.eclipse.swt.SWT.CURSOR_WAIT;
-import static org.eclipse.swt.SWT.READ_ONLY;
-import static org.eclipse.swt.SWT.RIGHT;
-
-import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
 import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.i18n.BaseMessages;
 import org.apache.hop.pipeline.PipelineMeta;
-import org.apache.hop.pipeline.transform.ITransformDialog;
-import org.apache.hop.ui.core.PropsUi;
 import org.apache.hop.ui.core.dialog.BaseDialog;
-import org.apache.hop.ui.core.dialog.ErrorDialog;
-import org.apache.hop.ui.core.widget.ComboItems;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
+import org.apache.hop.ui.core.widget.ComboVar;
 import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
-import org.eclipse.swt.SWT;
-import org.eclipse.swt.custom.CCombo;
-import org.eclipse.swt.events.FocusEvent;
-import org.eclipse.swt.events.FocusListener;
-import org.eclipse.swt.graphics.Cursor;
-import org.eclipse.swt.layout.FormAttachment;
-import org.eclipse.swt.layout.FormData;
-import org.eclipse.swt.widgets.Button;
-import org.eclipse.swt.widgets.Label;
-import org.eclipse.swt.widgets.Listener;
+import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Control;
 import org.eclipse.swt.widgets.Shell;
 
-public class DetectLanguageDialog extends BaseTransformDialog implements 
ITransformDialog {
+public class DetectLanguageDialog extends BaseTransformDialog {
   private static final Class<?> PKG = DetectLanguageDialog.class;
-  private boolean gotPreviousFields = false;
-
-  private CCombo wCorpusFieldName;
-  private Button wParallelism;
 
   private final DetectLanguageMeta input;
+  private GuiCompositeWidgets widgets;
 
   public DetectLanguageDialog(
       Shell parent,
@@ -69,82 +49,66 @@ public class DetectLanguageDialog extends 
BaseTransformDialog implements ITransf
   @Override
   public String open() {
     createShell(BaseMessages.getString(PKG, 
"DetectLanguageDialog.Shell.Title"));
-
     buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build();
 
-    Listener lsMod = e -> input.setChanged();
     changed = input.hasChanged();
 
-    // CorpusFieldName field
-    Label wlCorpusFieldName = new Label(shell, RIGHT);
-    wlCorpusFieldName.setText(getString(PKG, 
"DetectLanguageDialog.CorpusFieldName.Label"));
-    PropsUi.setLook(wlCorpusFieldName);
-    FormData fdlCorpusFieldName = new FormData();
-    fdlCorpusFieldName.left = new FormAttachment(0, 0);
-    fdlCorpusFieldName.right = new FormAttachment(middle, -margin);
-    fdlCorpusFieldName.top = new FormAttachment(wSpacer, margin);
-    wlCorpusFieldName.setLayoutData(fdlCorpusFieldName);
-
-    wCorpusFieldName = new CCombo(shell, BORDER | READ_ONLY);
-    PropsUi.setLook(wCorpusFieldName);
-    wCorpusFieldName.addListener(SWT.Modify, lsMod);
-    FormData fdCorpusFieldName = new FormData();
-    fdCorpusFieldName.left = new FormAttachment(middle, 0);
-    fdCorpusFieldName.top = new FormAttachment(wSpacer, margin);
-    fdCorpusFieldName.right = new FormAttachment(100, 0);
-    wCorpusFieldName.setLayoutData(fdCorpusFieldName);
-    wCorpusFieldName.addFocusListener(
-        new FocusListener() {
-          @Override
-          public void focusLost(FocusEvent e) {}
-
+    widgets =
+        GuiCompositeWidgets.addScrolledComposite(
+            shell,
+            variables,
+            wTransformName,
+            wOk,
+            DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID,
+            input);
+    widgets.setWidgetsListener(
+        new GuiCompositeWidgetsAdapter() {
           @Override
-          public void focusGained(FocusEvent e) {
-            Cursor busy = new Cursor(shell.getDisplay(), CURSOR_WAIT);
-            shell.setCursor(busy);
-            get();
-            shell.setCursor(null);
-            busy.dispose();
+          public void widgetModified(
+              GuiCompositeWidgets compositeWidgets, Control changedWidget, 
String widgetId) {
+            input.setChanged();
           }
         });
 
-    // parallelism
-    Label wlParallelism = new Label(shell, RIGHT);
-    wlParallelism.setText(getString(PKG, 
"DetectLanguageDialog.Parallelism.Label"));
-    PropsUi.setLook(wlParallelism);
-    FormData fdlParallelism = new FormData();
-    fdlParallelism.left = new FormAttachment(0, 0);
-    fdlParallelism.top = new FormAttachment(wCorpusFieldName, margin);
-    fdlParallelism.right = new FormAttachment(middle, -margin);
-    wlParallelism.setLayoutData(fdlParallelism);
-
-    wParallelism = new Button(shell, CHECK);
-    wParallelism.setSelection(input.isParallelism());
-    PropsUi.setLook(wParallelism);
-    wParallelism.setToolTipText(getString(PKG, 
"DetectLanguageDialog.Parallelism.Tooltip"));
-    FormData fdParallelism = new FormData();
-    fdParallelism.left = new FormAttachment(middle, 0);
-    fdParallelism.top = new FormAttachment(wCorpusFieldName, margin);
-    fdParallelism.right = new FormAttachment(100, 0);
-    wParallelism.setLayoutData(fdParallelism);
-    wParallelism.addListener(SWT.Selection, lsMod);
-
-    getData();
+    setFieldComboValues();
+    // Filling the field combo is not a change
     input.setChanged(changed);
+
     focusTransformName();
     BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
-
     return transformName;
   }
 
-  /** Copy information from the meta-data input to the dialog fields. */
-  public void getData() {
-
-    if (input.getCorpusField() != null) {
-      wCorpusFieldName.setText(input.getCorpusField());
+  /**
+   * Stream field names cannot come from {@code comboValuesMethod}, which is 
handed only a log
+   * channel and a metadata provider, so the field combo is filled once the 
widgets exist. The
+   * configured field stays when the previous fields cannot be read.
+   */
+  private void setFieldComboValues() {
+    try {
+      IRowMeta fields = pipelineMeta.getPrevTransformFields(variables, 
transformName);
+      String[] names = fields == null ? new String[0] : fields.getFieldNames();
+      widgets.setComboValues(DetectLanguageMeta.WIDGET_CORPUS_FIELD, names);
+      // Setting items clears the widget's text, so put the configured field 
back afterwards.
+      if (!Utils.isEmpty(input.getCorpusField())) {
+        setComboText(DetectLanguageMeta.WIDGET_CORPUS_FIELD, 
input.getCorpusField());
+      }
+    } catch (Exception e) {
+      LogChannel.UI.logError(
+          BaseMessages.getString(PKG, 
"DetectLanguageDialog.FailedToGetFields.DialogMessage"), e);
     }
+  }
 
-    wParallelism.setSelection(input.isParallelism());
+  private void setComboText(String widgetId, String text) {
+    Control control = widgets.getWidgetsMap().get(widgetId);
+    if (control == null || control.isDisposed()) {
+      return;
+    }
+    if (control instanceof ComboVar comboVar) {
+      comboVar.setText(text);
+    } else if (control instanceof Combo combo) {
+      combo.setText(text);
+    }
   }
 
   private void cancel() {
@@ -154,32 +118,12 @@ public class DetectLanguageDialog extends 
BaseTransformDialog implements ITransf
   }
 
   private void ok() {
-    if (isEmpty(wTransformName.getText())) {
+    if (Utils.isEmpty(wTransformName.getText())) {
       return;
     }
-    input.setCorpusField(wCorpusFieldName.getText());
-    input.setParallelism(wParallelism.getSelection());
-
-    transformName = wTransformName.getText(); // return value
-
+    widgets.getWidgetsContents(input, 
DetectLanguageMeta.GUI_PLUGIN_ELEMENT_PARENT_ID);
+    transformName = wTransformName.getText();
+    input.setChanged();
     dispose();
   }
-
-  private void get() {
-    if (!gotPreviousFields) {
-      try {
-        IRowMeta r = pipelineMeta.getPrevTransformFields(variables, 
transformName);
-        if (r != null) {
-          ComboItems.setItemsKeepingText(wCorpusFieldName, r.getFieldNames());
-        }
-        gotPreviousFields = true;
-      } catch (HopException ke) {
-        new ErrorDialog(
-            shell,
-            getString(PKG, 
"DetectLanguageDialog.FailedToGetFields.DialogTitle"),
-            getString(PKG, 
"DetectLanguageDialog.FailedToGetFields.DialogMessage"),
-            ke);
-      }
-    }
-  }
 }
diff --git 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
index efb62eddbc..0f65485c4e 100644
--- 
a/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
+++ 
b/plugins/transforms/detectlanguage/src/main/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMeta.java
@@ -14,21 +14,34 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hop.pipeline.transforms.language;
 
 import static org.apache.hop.core.ICheckResult.TYPE_RESULT_ERROR;
 import static org.apache.hop.core.ICheckResult.TYPE_RESULT_OK;
+import static org.apache.hop.core.ICheckResult.TYPE_RESULT_WARNING;
 import static org.apache.hop.core.util.Utils.isEmpty;
 
+import com.github.pemistahl.lingua.api.Language;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hop.core.CheckResult;
 import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.logging.ILogChannel;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.core.row.value.ValueMetaNumber;
 import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.StringUtil;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.i18n.BaseMessages;
 import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -37,6 +50,9 @@ import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.BaseTransformMeta;
 import org.apache.hop.pipeline.transform.TransformMeta;
 
+@Getter
+@Setter
+@GuiPlugin
 @Transform(
     id = "DetectLanguage",
     image = "detectlanguage.svg",
@@ -48,18 +64,88 @@ import org.apache.hop.pipeline.transform.TransformMeta;
 public class DetectLanguageMeta extends BaseTransformMeta<DetectLanguage, 
DetectLanguageData> {
   private static final Class<?> PKG = DetectLanguageMeta.class;
 
+  public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = 
"DetectLanguage.Dialog";
+  public static final String WIDGET_CORPUS_FIELD = "corpusField";
+  public static final String WIDGET_PARALLELISM = "parallelism";
+  public static final String WIDGET_LOW_ACCURACY = "lowAccuracy";
+  public static final String WIDGET_LANGUAGES = "languages";
+
+  private static final String GROUP_OPTIONS = 
"i18n::DetectLanguage.Group.Options";
+  private static final String GROUP_LANGUAGES = 
"i18n::DetectLanguage.Group.Languages";
+
+  @GuiWidgetElement(
+      id = WIDGET_CORPUS_FIELD,
+      order = "0100",
+      type = GuiElementType.COMBO,
+      label = "i18n::DetectLanguageDialog.CorpusFieldName.Label",
+      toolTip = "i18n::DetectLanguageDialog.CorpusFieldName.Tooltip",
+      parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+      groupType = GuiWidgetGroupType.BOXES,
+      group = GROUP_OPTIONS,
+      groupOrder = "0100")
   @HopMetadataProperty(
       key = "corpusField",
       injectionKey = "FIELD",
       injectionKeyDescription = "DetectLanguage.Injection.CorpusField")
   private String corpusField;
 
+  @GuiWidgetElement(
+      id = WIDGET_PARALLELISM,
+      order = "0200",
+      type = GuiElementType.CHECKBOX,
+      label = "i18n::DetectLanguageDialog.Parallelism.Label",
+      toolTip = "i18n::DetectLanguageDialog.Parallelism.Tooltip",
+      parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+      groupType = GuiWidgetGroupType.BOXES,
+      group = GROUP_OPTIONS,
+      groupOrder = "0100")
   @HopMetadataProperty(
       key = "parallelism",
       injectionKey = "PARALLISM",
       injectionKeyDescription = "DetectLanguage.Injection.Parallelism")
   private boolean parallelism = false;
 
+  /**
+   * Lingua's low accuracy mode: only the trigram models are used. Faster and 
much lighter on
+   * memory, less accurate on short text. Off for pipelines saved before the 
option existed.
+   */
+  @GuiWidgetElement(
+      id = WIDGET_LOW_ACCURACY,
+      order = "0300",
+      type = GuiElementType.CHECKBOX,
+      label = "i18n::DetectLanguageDialog.LowAccuracy.Label",
+      toolTip = "i18n::DetectLanguageDialog.LowAccuracy.Tooltip",
+      parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+      groupType = GuiWidgetGroupType.BOXES,
+      group = GROUP_OPTIONS,
+      groupOrder = "0100")
+  @HopMetadataProperty(
+      key = "lowAccuracy",
+      injectionKey = "LOW_ACCURACY",
+      injectionKeyDescription = "DetectLanguage.Injection.LowAccuracy")
+  private boolean lowAccuracy = false;
+
+  /**
+   * The languages the transform may answer. Empty means every spoken language 
Lingua knows, which
+   * is what the transform did before the option existed.
+   */
+  @GuiWidgetElement(
+      id = WIDGET_LANGUAGES,
+      order = "0400",
+      type = GuiElementType.TABLE,
+      toolTip = "i18n::DetectLanguageDialog.Languages.Tooltip",
+      parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+      groupType = GuiWidgetGroupType.BOXES,
+      group = GROUP_LANGUAGES,
+      groupOrder = "0200",
+      tableRows = 8)
+  @HopMetadataProperty(
+      key = "language",
+      groupKey = "languages",
+      injectionGroupKey = "LANGUAGES",
+      injectionGroupDescription = "DetectLanguage.Injection.Languages")
+  private List<DetectLanguageChoice> languages = new ArrayList<>();
+
   public DetectLanguageMeta() {
     super();
   }
@@ -67,6 +153,77 @@ public class DetectLanguageMeta extends 
BaseTransformMeta<DetectLanguage, Detect
   @Override
   public void setDefault() {
     parallelism = false;
+    lowAccuracy = false;
+    languages = new ArrayList<>();
+  }
+
+  /** The names offered in the languages table: every language Lingua can 
detect. */
+  public List<String> languageNames(ILogChannel log, IHopMetadataProvider 
metadataProvider) {
+    return Language.all().stream().map(Language::name).sorted().toList();
+  }
+
+  /**
+   * Resolves the configured languages.
+   *
+   * @return the languages to detect, without duplicates and in the configured 
order; empty when
+   *     none is configured, which means all spoken languages
+   * @throws HopException when a name is not a Lingua language or when exactly 
one language is
+   *     configured: detection needs at least two to choose from
+   */
+  public List<Language> resolveLanguages(IVariables variables) throws 
HopException {
+    return resolveLanguages(variables, null);
+  }
+
+  /**
+   * Resolves the configured languages, leaving out the names that still hold 
a variable.
+   *
+   * @param unresolved when not null, a name that still holds a variable after 
resolving is added
+   *     here instead of failing: Verify does not know the values a pipeline 
only gets when it runs
+   */
+  List<Language> resolveLanguages(IVariables variables, List<String> 
unresolved)
+      throws HopException {
+    List<Language> resolved = new ArrayList<>();
+    if (languages == null) {
+      return resolved;
+    }
+    for (DetectLanguageChoice choice : languages) {
+      String name = choice == null ? null : choice.getName();
+      if (variables != null) {
+        name = variables.resolve(name);
+      }
+      if (StringUtils.isBlank(name)) {
+        continue;
+      }
+      if (unresolved != null && StringUtil.containsVariableToken(name)) {
+        unresolved.add(name.trim());
+        continue;
+      }
+      Language language = findLanguage(name.trim());
+      if (language == null) {
+        throw new HopException(
+            BaseMessages.getString(PKG, 
"DetectLanguage.Error.UnknownLanguage", name.trim()));
+      }
+      if (!resolved.contains(language)) {
+        resolved.add(language);
+      }
+    }
+    // With names still to be resolved, the number of languages is not known 
yet
+    if (resolved.size() == 1 && (unresolved == null || unresolved.isEmpty())) {
+      throw new HopException(
+          BaseMessages.getString(
+              PKG, "DetectLanguage.Error.SingleLanguage", 
resolved.get(0).name()));
+    }
+    return resolved;
+  }
+
+  private static Language findLanguage(String name) {
+    String upper = name.toUpperCase(Locale.ROOT);
+    for (Language language : Language.all()) {
+      if (language.name().equals(upper)) {
+        return language;
+      }
+    }
+    return null;
   }
 
   @Override
@@ -77,7 +234,6 @@ public class DetectLanguageMeta extends 
BaseTransformMeta<DetectLanguage, Detect
       TransformMeta nextTransform,
       IVariables variables,
       IHopMetadataProvider metadataProvider) {
-
     // name = getParentTransformMeta().getName()
     valueMetaString(r, name, "detected_language");
     valueMetaNumber(r, name, "detected_language_confidence");
@@ -107,7 +263,6 @@ public class DetectLanguageMeta extends 
BaseTransformMeta<DetectLanguage, Detect
       IVariables variables,
       IHopMetadataProvider metadataProvider) {
     CheckResult cr;
-
     if (isEmpty(corpusField)) {
       cr =
           new CheckResult(
@@ -123,6 +278,33 @@ public class DetectLanguageMeta extends 
BaseTransformMeta<DetectLanguage, Detect
     }
     remarks.add(cr);
 
+    try {
+      List<String> unresolved = new ArrayList<>();
+      List<Language> resolved = resolveLanguages(variables, unresolved);
+      if (!unresolved.isEmpty()) {
+        cr =
+            new CheckResult(
+                TYPE_RESULT_WARNING,
+                BaseMessages.getString(
+                    PKG,
+                    
"DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime",
+                    String.join(", ", unresolved)),
+                transformMeta);
+      } else {
+        cr =
+            new CheckResult(
+                TYPE_RESULT_OK,
+                resolved.isEmpty()
+                    ? BaseMessages.getString(PKG, 
"DetectLanguageMeta.CheckResult.AllLanguages")
+                    : BaseMessages.getString(
+                        PKG, "DetectLanguageMeta.CheckResult.LanguagesOK", 
resolved.size()),
+                transformMeta);
+      }
+    } catch (HopException e) {
+      cr = new CheckResult(TYPE_RESULT_ERROR, e.getMessage().trim(), 
transformMeta);
+    }
+    remarks.add(cr);
+
     // See if we have input streams leading to this transform!
     if (input.length > 0) {
       cr =
@@ -145,20 +327,4 @@ public class DetectLanguageMeta extends 
BaseTransformMeta<DetectLanguage, Detect
   public boolean supportsErrorHandling() {
     return true;
   }
-
-  public String getCorpusField() {
-    return corpusField;
-  }
-
-  public void setCorpusField(String corpusField) {
-    this.corpusField = corpusField;
-  }
-
-  public boolean isParallelism() {
-    return parallelism;
-  }
-
-  public void setParallelism(boolean parallelism) {
-    this.parallelism = parallelism;
-  }
 }
diff --git 
a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
 
b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
index be58a587f5..a319f4c8a0 100644
--- 
a/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
+++ 
b/plugins/transforms/detectlanguage/src/main/resources/org/apache/hop/pipeline/transforms/language/messages/messages_en_US.properties
@@ -40,3 +40,19 @@ 
DetectLanguageMeta.CheckResult.ReceivingInfoFromOtherTransforms=Transform is rec
 DetectLanguageMeta.CheckResult.ErrorOccurred=An error occurred:
 DetectLanguageDialog.Parallelism.Label=Use Parallelism
 DetectLanguageDialog.Parallelism.Tooltip=Process the rows in parallel
+DetectLanguageDialog.CorpusFieldName.Tooltip=The field with the text to examine
+DetectLanguageDialog.LowAccuracy.Label=Low accuracy mode
+DetectLanguageDialog.LowAccuracy.Tooltip=Use only the three-letter language 
models: faster and much lighter on memory, but less accurate. Meant for longer 
texts: short texts of a few words are often given the wrong language.
+DetectLanguageDialog.Languages.Tooltip=The languages the text can be in. Leave 
the list empty to detect every spoken language. A short list is faster and uses 
less memory; a text in a language that is not listed is given one of the listed 
languages or UNKNOWN.
+DetectLanguage.Group.Options=Options
+DetectLanguage.Group.Languages=Languages to detect
+DetectLanguage.Column.Language.Label=Language
+DetectLanguage.Injection.LowAccuracy=Low accuracy mode
+DetectLanguage.Injection.Languages=Languages to detect
+DetectLanguage.Injection.Language=Language name
+DetectLanguage.Error.UnknownLanguage=''{0}'' is not a language this transform 
can detect. Use a name from the list, for example ENGLISH.
+DetectLanguage.Error.SingleLanguage=Only one language is listed ({0}). List at 
least two languages, or leave the list empty to detect every spoken language.
+DetectLanguage.Error.DetectorNotBuilt=Unable to prepare the language detection
+DetectLanguageMeta.CheckResult.AllLanguages=Every spoken language is detected.
+DetectLanguageMeta.CheckResult.LanguagesOK={0} languages are detected.
+DetectLanguageMeta.CheckResult.LanguagesResolvedAtRuntime=These languages are 
only known when the pipeline runs: {0}
diff --git 
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
 
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
index 3806cae064..75ca03e304 100644
--- 
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
+++ 
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageAsyncTest.java
@@ -23,18 +23,15 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.RETURNS_SELF;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import com.github.pemistahl.lingua.api.LanguageDetector;
-import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
 import java.lang.reflect.Field;
 import java.util.List;
 import java.util.TreeMap;
@@ -48,12 +45,12 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.exception.HopTransformException;
 import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
 import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
-import org.mockito.MockedStatic;
 
 @Timeout(10)
 class DetectLanguageAsyncTest {
@@ -72,15 +69,14 @@ class DetectLanguageAsyncTest {
     PipelineMeta pipelineMeta = new PipelineMeta();
     transformMeta = spy(new TransformMeta("test", meta));
     pipelineMeta.addTransform(transformMeta);
-    try (MockedStatic<LanguageDetectorBuilder> factory =
-        mockStatic(LanguageDetectorBuilder.class)) {
-      LanguageDetectorBuilder builder = mock(LanguageDetectorBuilder.class, 
RETURNS_SELF);
-      LanguageDetector detector = mock(LanguageDetector.class);
-      
factory.when(LanguageDetectorBuilder::fromAllSpokenLanguages).thenReturn(builder);
-      when(builder.build()).thenReturn(detector);
-      
when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new 
TreeMap<>());
-      transform = spy(new DetectLanguage(transformMeta, meta, data, 0, 
pipelineMeta, null));
-    }
+    LanguageDetector detector = mock(LanguageDetector.class);
+    when(detector.computeLanguageConfidenceValues(anyString())).thenReturn(new 
TreeMap<>());
+    data.detector = detector;
+    data.corpusField = "input";
+    transform = spy(new DetectLanguage(transformMeta, meta, data, 0, 
pipelineMeta, null));
+    RowMeta inputRowMeta = new RowMeta();
+    inputRowMeta.addValueMeta(new ValueMetaString("input"));
+    transform.setInputRowMeta(inputRowMeta);
     transform.first = false;
     data.indexOfCorpusField = 0;
     data.indexOfDetectedLanguage = 1;
diff --git 
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
 
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
index e4cda78ed1..c13196035c 100644
--- 
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
+++ 
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageMetaTest.java
@@ -14,15 +14,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hop.pipeline.transforms.language;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.github.pemistahl.lingua.api.Language;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
 import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.w3c.dom.Node;
 
 class DetectLanguageMetaTest {
   @RegisterExtension
@@ -30,11 +43,170 @@ class DetectLanguageMetaTest {
 
   @Test
   void testLoadSave() throws HopException {
-    List<String> attributes = List.of("corpusField", "parallelism");
+    List<String> attributes = List.of("corpusField", "parallelism", 
"lowAccuracy");
 
+    // the languages list has its own round trip test below
     LoadSaveTester<DetectLanguageMeta> loadSaveTester =
-        new LoadSaveTester<>(DetectLanguageMeta.class, attributes);
+        new LoadSaveTester<>(
+            DetectLanguageMeta.class,
+            attributes,
+            List.of("languages"),
+            new HashMap<>(),
+            new HashMap<>(),
+            new HashMap<>(),
+            new HashMap<>(),
+            null);
 
     loadSaveTester.testSerialization();
   }
+
+  /** A pipeline saved before the language list and the accuracy mode existed. 
*/
+  @Test
+  void aTransformSavedBeforeTheNewOptionsKeepsItsBehavior() throws Exception {
+    DetectLanguageMeta meta =
+        load(
+            
"<transform><corpusField>comment</corpusField><parallelism>Y</parallelism></transform>");
+
+    assertEquals("comment", meta.getCorpusField());
+    assertTrue(meta.isParallelism());
+    assertFalse(meta.isLowAccuracy());
+    assertTrue(meta.getLanguages().isEmpty());
+    // no languages listed: every spoken language, as before
+    assertTrue(meta.resolveLanguages(new Variables()).isEmpty());
+  }
+
+  @Test
+  void theNewOptionsSurviveAnXmlRoundTrip() throws Exception {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.setCorpusField("comment");
+    meta.setLowAccuracy(true);
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+    meta.getLanguages().add(new DetectLanguageChoice("FRENCH"));
+
+    DetectLanguageMeta copy = load("<transform>" + meta.getXml() + 
"</transform>");
+
+    assertEquals("comment", copy.getCorpusField());
+    assertFalse(copy.isParallelism());
+    assertTrue(copy.isLowAccuracy());
+    assertEquals(
+        List.of("ENGLISH", "FRENCH"),
+        
copy.getLanguages().stream().map(DetectLanguageChoice::getName).toList());
+  }
+
+  @Test
+  void languagesResolveWithVariablesAnyCaseAndNoDuplicates() throws Exception {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.getLanguages().add(new DetectLanguageChoice("english"));
+    meta.getLanguages().add(new DetectLanguageChoice(" ${SECOND} "));
+    meta.getLanguages().add(new DetectLanguageChoice(""));
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+    Variables variables = new Variables();
+    variables.setVariable("SECOND", "French");
+
+    assertEquals(List.of(Language.ENGLISH, Language.FRENCH), 
meta.resolveLanguages(variables));
+  }
+
+  @Test
+  void anUnknownLanguageIsRefused() {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+    meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+
+    HopException e = assertThrows(HopException.class, () -> 
meta.resolveLanguages(new Variables()));
+    assertTrue(e.getMessage().contains("KLINGON"));
+  }
+
+  @Test
+  void aSingleLanguageIsRefused() {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+
+    HopException e = assertThrows(HopException.class, () -> 
meta.resolveLanguages(new Variables()));
+    assertTrue(e.getMessage().contains("ENGLISH"));
+  }
+
+  @Test
+  void theLanguageListOffersEveryLinguaLanguage() {
+    List<String> names = new DetectLanguageMeta().languageNames(null, null);
+
+    assertEquals(Language.all().size(), names.size());
+    assertTrue(names.contains("ENGLISH"));
+    assertFalse(names.contains("UNKNOWN"));
+  }
+
+  @Test
+  void checkReportsABadLanguageList() {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.setCorpusField("comment");
+    meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+
+    List<ICheckResult> remarks = new ArrayList<>();
+    TransformMeta transformMeta = new TransformMeta();
+    transformMeta.setName("Detect");
+    meta.check(
+        remarks,
+        null,
+        transformMeta,
+        null,
+        new String[] {"in"},
+        new String[0],
+        null,
+        new Variables(),
+        new MemoryMetadataProvider());
+
+    assertTrue(
+        remarks.stream()
+            .anyMatch(
+                r ->
+                    r.getType() == ICheckResult.TYPE_RESULT_ERROR
+                        && r.getText().contains("KLINGON")));
+  }
+
+  /** A language from a variable that only gets a value when the pipeline runs 
is not an error. */
+  @Test
+  void checkLeavesLanguagesFromRuntimeVariablesToTheRun() {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.setCorpusField("comment");
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+    meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}"));
+
+    List<ICheckResult> remarks = new ArrayList<>();
+    TransformMeta transformMeta = new TransformMeta();
+    transformMeta.setName("Detect");
+    meta.check(
+        remarks,
+        null,
+        transformMeta,
+        null,
+        new String[] {"in"},
+        new String[0],
+        null,
+        new Variables(),
+        new MemoryMetadataProvider());
+
+    assertTrue(remarks.stream().noneMatch(r -> r.getType() == 
ICheckResult.TYPE_RESULT_ERROR));
+    assertTrue(
+        remarks.stream()
+            .anyMatch(
+                r ->
+                    r.getType() == ICheckResult.TYPE_RESULT_WARNING
+                        && r.getText().contains("${SECOND_LANGUAGE}")));
+  }
+
+  @Test
+  void runningWithAnUnresolvedLanguageStillFails() {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+    meta.getLanguages().add(new DetectLanguageChoice("${SECOND_LANGUAGE}"));
+
+    assertThrows(HopException.class, () -> meta.resolveLanguages(new 
Variables()));
+  }
+
+  private static DetectLanguageMeta load(String xml) throws Exception {
+    Node node = XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), 
"transform");
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.loadXml(node, new MemoryMetadataProvider());
+    return meta;
+  }
 }
diff --git 
a/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
 
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
new file mode 100644
index 0000000000..6f936af4e2
--- /dev/null
+++ 
b/plugins/transforms/detectlanguage/src/test/java/org/apache/hop/pipeline/transforms/language/DetectLanguageTest.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.language;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import com.github.pemistahl.lingua.api.Language;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class DetectLanguageTest {
+
+  private static final String ENGLISH_TEXT =
+      "The weather is lovely today and we are going for a long walk in the 
park";
+  private static final String FRENCH_TEXT =
+      "Il fait très beau aujourd'hui et nous allons nous promener dans le 
parc";
+  private static final String GERMAN_TEXT =
+      "Das Wetter ist heute sehr schön und wir gehen im Park spazieren";
+
+  @RegisterExtension
+  static RestoreHopEngineEnvironmentExtension env = new 
RestoreHopEngineEnvironmentExtension();
+
+  private TransformMockHelper<DetectLanguageMeta, DetectLanguageData> 
mockHelper;
+
+  /** Disposed after every test, also a failed one: the Lingua models live in 
static maps. */
+  private final List<TestDetectLanguage> transforms = new ArrayList<>();
+
+  @BeforeEach
+  void setUp() {
+    mockHelper =
+        new TransformMockHelper<>(
+            "Detect language", DetectLanguageMeta.class, 
DetectLanguageData.class);
+    when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(mockHelper.iLogChannel);
+    when(mockHelper.pipeline.isRunning()).thenReturn(true);
+    when(mockHelper.transformMeta.getCopies(any())).thenReturn(1);
+    when(mockHelper.transformMeta.isPartitioned()).thenReturn(false);
+    when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(false);
+  }
+
+  @AfterEach
+  void tearDown() {
+    for (TestDetectLanguage transform : transforms) {
+      transform.dispose();
+    }
+    mockHelper.cleanUp();
+  }
+
+  /** Without options the detector is what the transform always built. */
+  @Test
+  void theDefaultDetectorCoversEverySpokenLanguageWithHighAccuracy() {
+    LanguageDetectorBuilder builder = DetectLanguage.createBuilder(List.of(), 
false);
+
+    assertEquals(Language.allSpokenOnes(), builder.getLanguages$lingua());
+    assertFalse(builder.isLowAccuracyModeEnabled$lingua());
+    assertTrue(builder.isEveryLanguageModelPreloaded$lingua());
+  }
+
+  @Test
+  void theConfiguredLanguagesAndAccuracyModeReachTheDetector() {
+    LanguageDetectorBuilder builder =
+        DetectLanguage.createBuilder(List.of(Language.ENGLISH, 
Language.FRENCH), true);
+
+    assertEquals(List.of(Language.ENGLISH, Language.FRENCH), 
builder.getLanguages$lingua());
+    assertTrue(builder.isLowAccuracyModeEnabled$lingua());
+  }
+
+  @Test
+  void rowsGetTheirLanguageAndBlankTextsPassAsUnknown() throws Exception {
+    DetectLanguage transform = createTransform(meta(false, false));
+    assertTrue(transform.init());
+
+    BlockingRowSet output =
+        run(transform, row(1L, ENGLISH_TEXT), row(2L, null), row(3L, 
FRENCH_TEXT), row(4L, "   "));
+
+    Object[] english = output.getRowWait(1, TimeUnit.SECONDS);
+    assertEquals(1L, english[0]);
+    assertEquals("ENGLISH", english[2]);
+    assertNotNull(english[3]);
+
+    // a null or blank text used to drop the row
+    Object[] empty = output.getRowWait(1, TimeUnit.SECONDS);
+    assertEquals(2L, empty[0]);
+    assertEquals("UNKNOWN", empty[2]);
+    assertNull(empty[3]);
+
+    assertEquals("FRENCH", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+
+    Object[] blank = output.getRowWait(1, TimeUnit.SECONDS);
+    assertEquals(4L, blank[0]);
+    assertEquals("UNKNOWN", blank[2]);
+    assertNull(blank[3]);
+
+    assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+  }
+
+  @Test
+  void theCorpusFieldCanComeFromAVariable() throws Exception {
+    DetectLanguageMeta meta = meta(false, false);
+    meta.setCorpusField("${TEXT_FIELD}");
+    DetectLanguage transform = createTransform(meta);
+    assertTrue(transform.init());
+    transform.setVariable("TEXT_FIELD", "comment");
+
+    BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT));
+
+    assertEquals("GERMAN", output.getRowWait(1, TimeUnit.SECONDS)[2]);
+  }
+
+  @Test
+  void lowAccuracyModeFillsBothFields() throws Exception {
+    DetectLanguage transform = createTransform(meta(false, true));
+    assertTrue(transform.init());
+
+    BlockingRowSet output = run(transform, row(1L, GERMAN_TEXT));
+
+    Object[] german = output.getRowWait(1, TimeUnit.SECONDS);
+    assertEquals("GERMAN", german[2]);
+    assertNotNull(german[3]);
+  }
+
+  @Test
+  void parallelModeWritesEveryRow() throws Exception {
+    DetectLanguage transform = createTransform(meta(true, false));
+    assertTrue(transform.init());
+
+    BlockingRowSet output =
+        run(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L, 
GERMAN_TEXT));
+
+    Map<Object, Object> languageById = new HashMap<>();
+    for (int i = 0; i < 3; i++) {
+      Object[] row = output.getRowWait(5, TimeUnit.SECONDS);
+      assertNotNull(row);
+      languageById.put(row[0], row[2]);
+    }
+    assertEquals(Map.of(1L, "ENGLISH", 2L, "FRENCH", 3L, "GERMAN"), 
languageById);
+  }
+
+  /** A failing parallel job used to leave the transform waiting forever at 
the end. */
+  @Test
+  void aFailingParallelJobStopsTheTransform() {
+    DetectLanguage transform = createTransform(meta(true, false), 
DetectLanguageTest::noRoom);
+    assertTrue(transform.init());
+
+    HopException e =
+        assertTimeoutPreemptively(
+            Duration.ofSeconds(30),
+            () -> assertThrows(HopException.class, () -> run(transform, 
row(1L, ENGLISH_TEXT))));
+    assertTrue(e.getMessage().contains("no room for this row"));
+  }
+
+  @Test
+  void aFailingParallelRowGoesToTheErrorHop() throws Exception {
+    when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true);
+    TestDetectLanguage transform = createTransform(meta(true, false), 
DetectLanguageTest::noRoom);
+    assertTrue(transform.init());
+
+    assertTimeoutPreemptively(
+        Duration.ofSeconds(30), () -> run(transform, row(1L, ENGLISH_TEXT), 
row(2L, FRENCH_TEXT)));
+
+    List<ErrorRow> errorRows = new ArrayList<>(transform.errorRows);
+    assertEquals(2, errorRows.size());
+    for (ErrorRow errorRow : errorRows) {
+      assertEquals("comment", errorRow.fieldNames());
+      assertEquals("DetectLanguage001", errorRow.errorCodes());
+    }
+  }
+
+  /** An Error says nothing about the row: it stops the transform, also with 
error handling on. */
+  @Test
+  void anErrorInAParallelJobStopsTheTransform() {
+    when(mockHelper.transformMeta.isDoingErrorHandling()).thenReturn(true);
+    TestDetectLanguage transform =
+        createTransform(
+            meta(true, false),
+            (rowMeta, row) -> {
+              throw new NoClassDefFoundError("simulated failure");
+            });
+    assertTrue(transform.init());
+
+    HopException e =
+        assertTimeoutPreemptively(
+            Duration.ofSeconds(30),
+            () -> assertThrows(HopException.class, () -> run(transform, 
row(1L, ENGLISH_TEXT))));
+    assertTrue(e.getMessage().contains("simulated failure"));
+    assertTrue(transform.errorRows.isEmpty());
+  }
+
+  /**
+   * After an error or a stop the transform is disposed without reaching the 
end of its input.
+   * Parallel jobs that still run use the detector, so dispose waits for them.
+   */
+  @Test
+  void disposeWaitsForTheParallelJobsStillRunning() throws Exception {
+    CountDownLatch writing = new CountDownLatch(1);
+    CountDownLatch release = new CountDownLatch(1);
+    TestDetectLanguage transform =
+        createTransform(
+            meta(true, false),
+            (rowMeta, row) -> {
+              writing.countDown();
+              awaitQuietly(release);
+            });
+    assertTrue(transform.init());
+
+    connect(transform, row(1L, ENGLISH_TEXT), row(2L, FRENCH_TEXT), row(3L, 
GERMAN_TEXT));
+    for (int i = 0; i < 3; i++) {
+      assertTrue(transform.processRow());
+    }
+    assertTrue(writing.await(30, TimeUnit.SECONDS));
+
+    // What RunThread does when processRow fails or the pipeline is stopped
+    transform.stopAll();
+    Thread disposer = new Thread(transform::dispose, "dispose");
+    disposer.start();
+    disposer.join(300);
+    assertTrue(disposer.isAlive(), "dispose returned while a job was still 
running");
+    assertNotNull(transform.getData().detector);
+
+    release.countDown();
+    disposer.join(30_000);
+    assertFalse(disposer.isAlive());
+    assertNull(transform.getData().detector);
+    assertTrue(transform.errorRows.isEmpty());
+  }
+
+  @Test
+  void anUnknownLanguageFailsAtStart() {
+    DetectLanguageMeta meta = meta(false, false);
+    meta.getLanguages().add(new DetectLanguageChoice("KLINGON"));
+
+    assertFalse(createTransform(meta).init());
+  }
+
+  private static DetectLanguageMeta meta(boolean parallelism, boolean 
lowAccuracy) {
+    DetectLanguageMeta meta = new DetectLanguageMeta();
+    meta.setCorpusField("comment");
+    meta.setParallelism(parallelism);
+    meta.setLowAccuracy(lowAccuracy);
+    // three languages keep the models small; the default is covered by the 
builder test
+    meta.getLanguages().add(new DetectLanguageChoice("ENGLISH"));
+    meta.getLanguages().add(new DetectLanguageChoice("FRENCH"));
+    meta.getLanguages().add(new DetectLanguageChoice("GERMAN"));
+    return meta;
+  }
+
+  private static Object[] row(Long id, String comment) {
+    return new Object[] {id, comment};
+  }
+
+  private static void noRoom(IRowMeta rowMeta, Object[] row) throws 
HopTransformException {
+    throw new HopTransformException("no room for this row");
+  }
+
+  private static void awaitQuietly(CountDownLatch latch) {
+    try {
+      latch.await(30, TimeUnit.SECONDS);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  /** Connects an input holding the rows, and an output, to the transform. */
+  private static BlockingRowSet connect(DetectLanguage transform, Object[]... 
rows) {
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("id"));
+    rowMeta.addValueMeta(new ValueMetaString("comment"));
+
+    BlockingRowSet input = new BlockingRowSet(100);
+    input.setThreadNameFromToCopy("previous", 0, "Detect language", 0);
+    input.setRowMeta(rowMeta);
+    for (Object[] row : rows) {
+      input.putRow(rowMeta, row);
+    }
+    input.setDone();
+
+    BlockingRowSet output = new BlockingRowSet(100);
+    output.setThreadNameFromToCopy("Detect language", 0, "next", 0);
+
+    transform.addRowSetToInputRowSets(input);
+    transform.addRowSetToOutputRowSets(output);
+    return output;
+  }
+
+  /** Feeds the rows through the transform until it reports the end of its 
input. */
+  private static BlockingRowSet run(DetectLanguage transform, Object[]... rows)
+      throws HopException {
+    BlockingRowSet output = connect(transform, rows);
+    List<Boolean> results = new ArrayList<>();
+    boolean more = true;
+    while (more) {
+      more = transform.processRow();
+      results.add(more);
+    }
+    assertEquals(rows.length + 1, results.size());
+    return output;
+  }
+
+  private TestDetectLanguage createTransform(DetectLanguageMeta meta) {
+    return createTransform(meta, null);
+  }
+
+  private TestDetectLanguage createTransform(DetectLanguageMeta meta, RowWrite 
write) {
+    when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+    TestDetectLanguage transform = new TestDetectLanguage(mockHelper, meta, 
write);
+    transforms.add(transform);
+    return transform;
+  }
+
+  /** What a test's transform does with an output row instead of writing it. */
+  @FunctionalInterface
+  private interface RowWrite {
+    void write(IRowMeta rowMeta, Object[] row) throws HopTransformException;
+  }
+
+  private record ErrorRow(Object id, String fieldNames, String errorCodes) {}
+
+  /** The transform, with the writes a test controls and the error rows it 
sends. */
+  private static class TestDetectLanguage extends DetectLanguage {
+    private final RowWrite write;
+    final List<ErrorRow> errorRows = Collections.synchronizedList(new 
ArrayList<>());
+
+    TestDetectLanguage(
+        TransformMockHelper<DetectLanguageMeta, DetectLanguageData> helper,
+        DetectLanguageMeta meta,
+        RowWrite write) {
+      super(
+          helper.transformMeta,
+          meta,
+          new DetectLanguageData(),
+          0,
+          helper.pipelineMeta,
+          helper.pipeline);
+      this.write = write;
+    }
+
+    @Override
+    public void putRow(IRowMeta rowMeta, Object[] row) throws 
HopTransformException {
+      if (write == null) {
+        super.putRow(rowMeta, row);
+      } else {
+        write.write(rowMeta, row);
+      }
+    }
+
+    @Override
+    public void putError(
+        IRowMeta rowMeta,
+        Object[] row,
+        long nrErrors,
+        String errorDescriptions,
+        String fieldNames,
+        String errorCodes) {
+      errorRows.add(new ErrorRow(row[0], fieldNames, errorCodes));
+    }
+  }
+}

Reply via email to