This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new c86e1e7d OPENNLP-1659 - Enhancements for DownloadUtil
c86e1e7d is described below
commit c86e1e7deffd716f9881f88b07df249fc7c9852e
Author: Richard Zowalla <[email protected]>
AuthorDate: Tue Nov 26 20:10:16 2024 +0100
OPENNLP-1659 - Enhancements for DownloadUtil
- Make the BASE_URL and MODEL_URI_PATH configurable via System properties
in order to be backwards compatible if models are removed from the release area
of OpenNLP / ASF CDN
- Make the OPENNLP_DOWNLOAD_HOME configurable, i.e. for tests in CI
environments. This can also be useful for OpenNLP devs to avoid cleaning models
in their user home.
- Replace hard coded file separators for multi OS support
- Log a DEBUG message if the model file already exists and the download is
skipped.
- Adds a test case to verify that models are not downloaded twice (if they
exist) by testing for the related LOG output using LogCaptor.
- Replaces slf4j-simple with logback (from LogCaptor) for further testing
of specific output
---
.github/workflows/maven.yml | 2 +-
opennlp-tools/pom.xml | 34 ++++--
.../main/java/opennlp/tools/util/DownloadUtil.java | 48 +++++---
.../opennlp/tools/AbstractModelLoaderTest.java | 3 +-
.../tools/util/AbstractDownloadUtilTest.java | 79 +++++++++++++
.../tools/util/DownloadUtilDownloadTwiceTest.java | 97 ++++++++++++++++
.../java/opennlp/tools/util/DownloadUtilTest.java | 125 +++++++--------------
opennlp-tools/src/test/resources/logback-test.xml | 36 ++++++
pom.xml | 4 +-
9 files changed, 315 insertions(+), 113 deletions(-)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index e568f5d6..2655ece1 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -52,6 +52,6 @@ jobs:
distribution: temurin
java-version: ${{ matrix.java }}
- name: Build with Maven
- run: mvn -V clean test install --no-transfer-progress -Pjacoco
+ run: mvn -V clean test install --no-transfer-progress -Pjacoco -Pci
- name: Jacoco
run: mvn jacoco:report
diff --git a/opennlp-tools/pom.xml b/opennlp-tools/pom.xml
index 4453b396..8b82760f 100644
--- a/opennlp-tools/pom.xml
+++ b/opennlp-tools/pom.xml
@@ -57,12 +57,6 @@
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <scope>test</scope>
- </dependency>
-
<!-- JUnit5 extension used in CLITest to prevent System.exit(..) calls
terminating test runs -->
<dependency>
<groupId>com.ginsberg</groupId>
@@ -71,6 +65,13 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.github.hakky54</groupId>
+ <artifactId>logcaptor</artifactId>
+ <version>${logcaptor.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
@@ -121,7 +122,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <argLine>-Xmx2048m -Dorg.slf4j.simpleLogger.defaultLogLevel=off
-javaagent:${settings.localRepository}/com/ginsberg/junit5-system-exit/${junit5-system-exit.version}/junit5-system-exit-${junit5-system-exit.version}.jar</argLine>
+ <argLine>-Xmx2048m -DOPENNLP_DOWNLOAD_HOME=${opennlp.download.home}
-javaagent:${settings.localRepository}/com/ginsberg/junit5-system-exit/${junit5-system-exit.version}/junit5-system-exit-${junit5-system-exit.version}.jar</argLine>
<forkCount>${opennlp.forkCount}</forkCount>
<failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
<excludes>
@@ -131,10 +132,29 @@
</excludes>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${maven.failsafe.plugin}</version>
+ <configuration>
+ <argLine>-DOPENNLP_DOWNLOAD_HOME=${opennlp.download.home}</argLine>
+ </configuration>
+ </plugin>
</plugins>
</build>
+ <properties>
+ <opennlp.download.home>${user.home}</opennlp.download.home>
+ </properties>
+
<profiles>
+ <profile>
+ <id>ci</id>
+ <properties>
+
<opennlp.download.home>${project.build.directory}</opennlp.download.home>
+ </properties>
+ </profile>
<profile>
<id>jmh</id>
<dependencies>
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/DownloadUtil.java
b/opennlp-tools/src/main/java/opennlp/tools/util/DownloadUtil.java
index 76fb7bc6..11b328da 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/DownloadUtil.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/DownloadUtil.java
@@ -32,6 +32,7 @@ import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
@@ -72,18 +73,12 @@ public class DownloadUtil {
}
}
- private static final String BASE_URL = "https://dlcdn.apache.org/opennlp/";;
- private static final String MODELS_UD_MODELS_1_2 = "models/ud-models-1.2/";
+ private static final String BASE_URL =
+ System.getProperty("OPENNLP_DOWNLOAD_BASE_URL",
"https://dlcdn.apache.org/opennlp/";);
+ private static final String MODEL_URI_PATH =
+ System.getProperty("OPENNLP_DOWNLOAD_MODEL_PATH",
"models/ud-models-1.2/");
- public static final Map<String, Map<ModelType, String>> available_models;
-
- static {
- try {
- available_models = new DownloadParser(new URL(BASE_URL +
MODELS_UD_MODELS_1_2)).getAvailableModels();
- } catch (MalformedURLException e) {
- throw new RuntimeException(e);
- }
- }
+ private static Map<String, Map<ModelType, String>> availableModels;
/**
* Triggers a download for the specified {@link DownloadUtil.ModelType}.
@@ -98,14 +93,14 @@ public class DownloadUtil {
public static <T extends BaseModel> T downloadModel(String language,
ModelType modelType,
Class<T> type) throws
IOException {
- if (available_models.containsKey(language)) {
- final String url = (available_models.get(language).get(modelType));
+ if (getAvailableModels().containsKey(language)) {
+ final String url = (getAvailableModels().get(language).get(modelType));
if (url != null) {
return downloadModel(new URL(url), type);
}
}
- throw new IOException("Invalid model.");
+ throw new IOException("There is no model available: " + language + " " +
modelType.name);
}
/**
@@ -124,9 +119,15 @@ public class DownloadUtil {
*/
public static <T extends BaseModel> T downloadModel(URL url, Class<T> type)
throws IOException {
- final Path homeDirectory = Paths.get(System.getProperty("user.home") +
"/.opennlp/");
+ final Path homeDirectory =
Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+
if (!Files.isDirectory(homeDirectory)) {
- homeDirectory.toFile().mkdir();
+ try {
+ Files.createDirectories(homeDirectory);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
final String filename =
url.toString().substring(url.toString().lastIndexOf("/") + 1);
@@ -141,8 +142,10 @@ public class DownloadUtil {
validateModel(new URL(url + ".sha512"), localFile);
-
logger.debug("Download complete.");
+ } else {
+ System.out.println("Model file already exists. Skipping download.");
+ logger.debug("Model file '{}' already exists. Skipping download.",
filename);
}
try {
@@ -152,6 +155,17 @@ public class DownloadUtil {
}
}
+ public static Map<String, Map<ModelType, String>> getAvailableModels() {
+ if (availableModels == null) {
+ try {
+ availableModels = new DownloadParser(new URL(BASE_URL +
MODEL_URI_PATH)).getAvailableModels();
+ } catch (MalformedURLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return Collections.unmodifiableMap(availableModels);
+ }
+
/**
* Validates the downloaded model.
*
diff --git
a/opennlp-tools/src/test/java/opennlp/tools/AbstractModelLoaderTest.java
b/opennlp-tools/src/test/java/opennlp/tools/AbstractModelLoaderTest.java
index cc36db27..db0ae96d 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/AbstractModelLoaderTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/AbstractModelLoaderTest.java
@@ -36,7 +36,8 @@ public abstract class AbstractModelLoaderTest {
private static final String BASE_URL_MODELS_V15 =
"https://opennlp.sourceforge.net/models-1.5/";;
private static final String BASE_URL_MODELS_V183 =
"https://dlcdn.apache.org/opennlp/models/langdetect/1.8.3/";;
- protected static final Path OPENNLP_DIR =
Paths.get(System.getProperty("user.home") + "/.opennlp/");
+ protected static final Path OPENNLP_DIR =
Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
protected static final String VER = "1.2-2.5.0";
protected static final String BIN = ".bin";
protected static List<String> SUPPORTED_LANG_CODES = List.of(
diff --git
a/opennlp-tools/src/test/java/opennlp/tools/util/AbstractDownloadUtilTest.java
b/opennlp-tools/src/test/java/opennlp/tools/util/AbstractDownloadUtilTest.java
new file mode 100644
index 00000000..9fdde8b6
--- /dev/null
+++
b/opennlp-tools/src/test/java/opennlp/tools/util/AbstractDownloadUtilTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.BeforeAll;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+@EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+public abstract class AbstractDownloadUtilTest {
+
+ private static final String APACHE_CDN = "dlcdn.apache.org";
+
+ @BeforeAll
+ public static void cleanupWhenOnline() {
+ boolean isOnline;
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(APACHE_CDN, 80),
EnabledWhenCDNAvailable.TIMEOUT_MS);
+ isOnline = true;
+ } catch (IOException e) {
+ // Unreachable, unresolvable or timeout
+ isOnline = false;
+ }
+ // If CDN is available -> go cleanup in preparation of the actual tests
+ if (isOnline) {
+ wipeExistingModelFiles("-tokens-");
+ wipeExistingModelFiles("-sentence-");
+ wipeExistingModelFiles("-pos-");
+ wipeExistingModelFiles("-lemmas-");
+ }
+ }
+
+
+ /*
+ * Helper method that wipes out mode files if they exist on the text
execution env.
+ * Those model files are wiped from a hidden '.opennlp' subdirectory.
+ *
+ * Thereby, a clean download can be guaranteed - ín CDN is available and
test are executed.
+ */
+ private static void wipeExistingModelFiles(final String fragment) {
+ final Path dir = Paths.get(System.getProperty("OPENNLP_DOWNLOAD_HOME",
+ System.getProperty("user.home"))).resolve(".opennlp");
+ if (Files.exists(dir)) {
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir,
"*opennlp-*" + fragment + "*")) {
+ for (Path modelFileToWipe : stream) {
+ Files.deleteIfExists(modelFileToWipe);
+ }
+ } catch (IOException e) {
+ fail(e.getLocalizedMessage());
+ }
+ }
+ }
+
+}
diff --git
a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilDownloadTwiceTest.java
b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilDownloadTwiceTest.java
new file mode 100644
index 00000000..5f328b00
--- /dev/null
+++
b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilDownloadTwiceTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.IOException;
+import java.util.List;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import nl.altindag.log.LogCaptor;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.sentdetect.SentenceModel;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+public class DownloadUtilDownloadTwiceTest extends AbstractDownloadUtilTest {
+
+ /*
+ * Programmatic change to debug log to ensure that we can see log messages to
+ * confirm no duplicate download is happening
+ */
+ @BeforeAll
+ public static void prepare() {
+ LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+ Logger logger = context.getLogger("opennlp");
+ logger.setLevel(Level.DEBUG);
+ }
+
+ /*
+ * Programmatic restore the default log level (= OFF) after the test
+ */
+ @AfterAll
+ public static void cleanup() {
+ LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+ Logger logger = context.getLogger("opennlp");
+ logger.setLevel(Level.OFF);
+ }
+
+ @Test
+ public void testDownloadModelTwice() throws IOException {
+ try (LogCaptor logCaptor = LogCaptor.forClass(DownloadUtil.class)) {
+
+ DownloadUtil.downloadModel("de",
+ DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
+
+ assertEquals(2, logCaptor.getDebugLogs().size());
+ checkDebugLogsContainMessageFragment(logCaptor.getDebugLogs(), "Download
complete.");
+ logCaptor.clearLogs();
+
+ // try to download again
+ DownloadUtil.downloadModel("de",
+ DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
+ assertEquals(1, logCaptor.getDebugLogs().size());
+ checkDebugLogsContainMessageFragment(logCaptor.getDebugLogs(), "already
exists. Skipping download.");
+ logCaptor.clearLogs();
+
+ DownloadUtil.downloadModel("de",
+ DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
+ assertEquals(1, logCaptor.getDebugLogs().size());
+ checkDebugLogsContainMessageFragment(logCaptor.getDebugLogs(), "already
exists. Skipping download.");
+ logCaptor.clearLogs();
+
+ }
+ }
+
+ private void checkDebugLogsContainMessageFragment(List<String> debugLogs,
String message) {
+ for (String log : debugLogs) {
+ if (log.contains(message)) {
+ return;
+ }
+ }
+ throw new AssertionError("Expected message fragment not found in logs: " +
message);
+ }
+
+}
diff --git
a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
index 6ab0aa4c..b8a61f91 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
@@ -18,16 +18,9 @@
package opennlp.tools.util;
import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
import java.net.URL;
-import java.nio.file.DirectoryStream;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.stream.Stream;
-import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -42,58 +35,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-public class DownloadUtilTest {
-
- private static final String APACHE_CDN = "dlcdn.apache.org";
-
- @BeforeAll
- public static void cleanupWhenOnline() {
- boolean isOnline;
- try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(APACHE_CDN, 80),
EnabledWhenCDNAvailable.TIMEOUT_MS);
- isOnline = true;
- } catch (IOException e) {
- // Unreachable, unresolvable or timeout
- isOnline = false;
- }
- // If CDN is available -> go cleanup in preparation of the actual tests
- if (isOnline) {
- wipeExistingModelFiles("-tokens-");
- wipeExistingModelFiles("-sentence-");
- }
- }
-
- /*
- * Helper method that wipes out mode files if they exist on the text
execution env.
- * Those model files are wiped from the user's home hidden '.opennlp'
subdirectory.
- *
- * Thereby, a clean download can be guaranteed - ín CDN is available and
test are executed.
- */
- private static void wipeExistingModelFiles(final String fragment) {
- final String openNLPHomeDir = System.getProperty("user.home") +
"/.opennlp/";
- final Path dir = FileSystems.getDefault().getPath(openNLPHomeDir);
- if (Files.exists(dir)) {
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir,
"*opennlp-*" + fragment + "*")) {
- for (Path modelFileToWipe: stream) {
- Files.deleteIfExists(modelFileToWipe);
- }
- } catch (IOException e) {
- fail(e.getLocalizedMessage());
- }
- }
- }
+public class DownloadUtilTest extends AbstractDownloadUtilTest {
@ParameterizedTest(name = "Verify \"{0}\" sentence model")
@ValueSource(strings = {"en", "fr", "de", "it", "nl", "bg", "ca", "cs",
"da", "el",
- "es", "et", "eu", "fi", "hr", "hy", "is", "ka",
"kk", "ko",
- "lv", "no", "pl", "pt", "ro", "ru", "sk", "sl",
"sr", "sv",
- "tr", "uk"})
+ "es", "et", "eu", "fi", "hr", "hy", "is", "ka", "kk", "ko",
+ "lv", "no", "pl", "pt", "ro", "ru", "sk", "sl", "sr", "sv",
+ "tr", "uk"})
@EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
public void testDownloadModelByLanguage(String lang) throws IOException {
SentenceModel model = DownloadUtil.downloadModel(lang,
- DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
+ DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
assertNotNull(model);
assertEquals(lang, model.getLanguage());
assertTrue(model.isLoadedFromSerialized());
@@ -114,8 +67,8 @@ public class DownloadUtilTest {
@ValueSource(strings = {" ", "\t", "\n"})
public void testDownloadModelInvalid(String input) {
assertThrows(IOException.class, () -> DownloadUtil.downloadModel(
- input, DownloadUtil.ModelType.SENTENCE_DETECTOR,
SentenceModel.class),
- "Invalid model");
+ input, DownloadUtil.ModelType.SENTENCE_DETECTOR,
SentenceModel.class),
+ "Invalid model");
}
private static final DownloadUtil.ModelType MT_TOKENIZER =
DownloadUtil.ModelType.TOKENIZER;
@@ -123,38 +76,38 @@ public class DownloadUtilTest {
// Note: This needs to be public as JUnit 5 requires it like this.
public static Stream<Arguments> provideURLs() {
return Stream.of(
- Arguments.of("en",
DownloadUtil.available_models.get("en").get(MT_TOKENIZER)),
- Arguments.of("fr",
DownloadUtil.available_models.get("fr").get(MT_TOKENIZER)),
- Arguments.of("de",
DownloadUtil.available_models.get("de").get(MT_TOKENIZER)),
- Arguments.of("it",
DownloadUtil.available_models.get("it").get(MT_TOKENIZER)),
- Arguments.of("nl",
DownloadUtil.available_models.get("nl").get(MT_TOKENIZER)),
- Arguments.of("bg",
DownloadUtil.available_models.get("bg").get(MT_TOKENIZER)),
- Arguments.of("ca",
DownloadUtil.available_models.get("ca").get(MT_TOKENIZER)),
- Arguments.of("cs",
DownloadUtil.available_models.get("cs").get(MT_TOKENIZER)),
- Arguments.of("da",
DownloadUtil.available_models.get("da").get(MT_TOKENIZER)),
- Arguments.of("el",
DownloadUtil.available_models.get("el").get(MT_TOKENIZER)),
- Arguments.of("es",
DownloadUtil.available_models.get("es").get(MT_TOKENIZER)),
- Arguments.of("et",
DownloadUtil.available_models.get("et").get(MT_TOKENIZER)),
- Arguments.of("eu",
DownloadUtil.available_models.get("eu").get(MT_TOKENIZER)),
- Arguments.of("fi",
DownloadUtil.available_models.get("fi").get(MT_TOKENIZER)),
- Arguments.of("hr",
DownloadUtil.available_models.get("hr").get(MT_TOKENIZER)),
- Arguments.of("hy",
DownloadUtil.available_models.get("hy").get(MT_TOKENIZER)),
- Arguments.of("is",
DownloadUtil.available_models.get("is").get(MT_TOKENIZER)),
- Arguments.of("ka",
DownloadUtil.available_models.get("ka").get(MT_TOKENIZER)),
- Arguments.of("kk",
DownloadUtil.available_models.get("kk").get(MT_TOKENIZER)),
- Arguments.of("ko",
DownloadUtil.available_models.get("ko").get(MT_TOKENIZER)),
- Arguments.of("lv",
DownloadUtil.available_models.get("lv").get(MT_TOKENIZER)),
- Arguments.of("no",
DownloadUtil.available_models.get("no").get(MT_TOKENIZER)),
- Arguments.of("pl",
DownloadUtil.available_models.get("pl").get(MT_TOKENIZER)),
- Arguments.of("pt",
DownloadUtil.available_models.get("pt").get(MT_TOKENIZER)),
- Arguments.of("ro",
DownloadUtil.available_models.get("ro").get(MT_TOKENIZER)),
- Arguments.of("ru",
DownloadUtil.available_models.get("ru").get(MT_TOKENIZER)),
- Arguments.of("sk",
DownloadUtil.available_models.get("sk").get(MT_TOKENIZER)),
- Arguments.of("sl",
DownloadUtil.available_models.get("sl").get(MT_TOKENIZER)),
- Arguments.of("sr",
DownloadUtil.available_models.get("sr").get(MT_TOKENIZER)),
- Arguments.of("sv",
DownloadUtil.available_models.get("sv").get(MT_TOKENIZER)),
- Arguments.of("tr",
DownloadUtil.available_models.get("tr").get(MT_TOKENIZER)),
- Arguments.of("uk",
DownloadUtil.available_models.get("uk").get(MT_TOKENIZER))
+ Arguments.of("en",
DownloadUtil.getAvailableModels().get("en").get(MT_TOKENIZER)),
+ Arguments.of("fr",
DownloadUtil.getAvailableModels().get("fr").get(MT_TOKENIZER)),
+ Arguments.of("de",
DownloadUtil.getAvailableModels().get("de").get(MT_TOKENIZER)),
+ Arguments.of("it",
DownloadUtil.getAvailableModels().get("it").get(MT_TOKENIZER)),
+ Arguments.of("nl",
DownloadUtil.getAvailableModels().get("nl").get(MT_TOKENIZER)),
+ Arguments.of("bg",
DownloadUtil.getAvailableModels().get("bg").get(MT_TOKENIZER)),
+ Arguments.of("ca",
DownloadUtil.getAvailableModels().get("ca").get(MT_TOKENIZER)),
+ Arguments.of("cs",
DownloadUtil.getAvailableModels().get("cs").get(MT_TOKENIZER)),
+ Arguments.of("da",
DownloadUtil.getAvailableModels().get("da").get(MT_TOKENIZER)),
+ Arguments.of("el",
DownloadUtil.getAvailableModels().get("el").get(MT_TOKENIZER)),
+ Arguments.of("es",
DownloadUtil.getAvailableModels().get("es").get(MT_TOKENIZER)),
+ Arguments.of("et",
DownloadUtil.getAvailableModels().get("et").get(MT_TOKENIZER)),
+ Arguments.of("eu",
DownloadUtil.getAvailableModels().get("eu").get(MT_TOKENIZER)),
+ Arguments.of("fi",
DownloadUtil.getAvailableModels().get("fi").get(MT_TOKENIZER)),
+ Arguments.of("hr",
DownloadUtil.getAvailableModels().get("hr").get(MT_TOKENIZER)),
+ Arguments.of("hy",
DownloadUtil.getAvailableModels().get("hy").get(MT_TOKENIZER)),
+ Arguments.of("is",
DownloadUtil.getAvailableModels().get("is").get(MT_TOKENIZER)),
+ Arguments.of("ka",
DownloadUtil.getAvailableModels().get("ka").get(MT_TOKENIZER)),
+ Arguments.of("kk",
DownloadUtil.getAvailableModels().get("kk").get(MT_TOKENIZER)),
+ Arguments.of("ko",
DownloadUtil.getAvailableModels().get("ko").get(MT_TOKENIZER)),
+ Arguments.of("lv",
DownloadUtil.getAvailableModels().get("lv").get(MT_TOKENIZER)),
+ Arguments.of("no",
DownloadUtil.getAvailableModels().get("no").get(MT_TOKENIZER)),
+ Arguments.of("pl",
DownloadUtil.getAvailableModels().get("pl").get(MT_TOKENIZER)),
+ Arguments.of("pt",
DownloadUtil.getAvailableModels().get("pt").get(MT_TOKENIZER)),
+ Arguments.of("ro",
DownloadUtil.getAvailableModels().get("ro").get(MT_TOKENIZER)),
+ Arguments.of("ru",
DownloadUtil.getAvailableModels().get("ru").get(MT_TOKENIZER)),
+ Arguments.of("sk",
DownloadUtil.getAvailableModels().get("sk").get(MT_TOKENIZER)),
+ Arguments.of("sl",
DownloadUtil.getAvailableModels().get("sl").get(MT_TOKENIZER)),
+ Arguments.of("sr",
DownloadUtil.getAvailableModels().get("sr").get(MT_TOKENIZER)),
+ Arguments.of("sv",
DownloadUtil.getAvailableModels().get("sv").get(MT_TOKENIZER)),
+ Arguments.of("tr",
DownloadUtil.getAvailableModels().get("tr").get(MT_TOKENIZER)),
+ Arguments.of("uk",
DownloadUtil.getAvailableModels().get("uk").get(MT_TOKENIZER))
);
}
}
diff --git a/opennlp-tools/src/test/resources/logback-test.xml
b/opennlp-tools/src/test/resources/logback-test.xml
new file mode 100644
index 00000000..b3cbcf20
--- /dev/null
+++ b/opennlp-tools/src/test/resources/logback-test.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+
+ <appender name="consoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%date{HH:mm:ss.SSS} [%thread] %-5level
%class{36}.%method:%line - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="opennlp" level="off"/>
+
+ <root level="off">
+ <appender-ref ref="consoleAppender" />
+ </root>
+
+</configuration>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index c6964abc..3e3906b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,9 +175,11 @@
<onnxruntime.version>1.20.0</onnxruntime.version>
<slf4j.version>2.0.16</slf4j.version>
<log4j2.version>2.24.2</log4j2.version>
+ <logcaptor.version>2.10.0</logcaptor.version>
<jmh.version>1.37</jmh.version>
<classgraph.version>4.8.179</classgraph.version>
-
+
+
<opennlp.models.version>1.2.0</opennlp.models.version>
<opennlp.forkCount>1.0C</opennlp.forkCount>
<coveralls.maven.plugin>4.3.0</coveralls.maven.plugin>
