[
https://issues.apache.org/jira/browse/TIKA-2790?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16873640#comment-16873640
]
Hudson commented on TIKA-2790:
------------------------------
SUCCESS: Integrated in Jenkins build Tika-trunk #1670 (See
[https://builds.apache.org/job/Tika-trunk/1670/])
TIKA-2790 -- migrate lang id to an OpenNLP-based lang id in tika-eval;
(tallison:
[https://github.com/apache/tika/commit/86888d9cdf39455bf8bb8dda05e77e790749476b])
* (edit) tika-core/src/main/java/org/apache/tika/utils/ProcessUtils.java
* (add) tika-eval/src/main/resources/common_tokens/bos
* (add) tika-eval/src/main/resources/common_tokens/epo
* (add) tika-eval/src/main/resources/common_tokens/kir
* (add) tika-eval/src/main/resources/common_tokens/aze
* (edit) tika-parent/pom.xml
* (add) tika-eval/src/main/resources/common_tokens/srp
* (add) tika-eval/src/main/resources/common_tokens/snd
* (add) tika-eval/src/main/resources/common_tokens/fry
* (add) tika-eval/src/main/resources/common_tokens/cat
* (add) tika-eval/src/main/resources/common_tokens/ekk
* (add) tika-eval/src/main/resources/common_tokens/nld
* (add) tika-eval/.gitignore
* (add) tika-eval/src/main/resources/common_tokens/pnb
* (add) tika-eval/src/main/resources/common_tokens/sin
* (add) tika-eval/src/main/resources/common_tokens/hun
* (delete) tika-eval/src/main/resources/common_tokens/ru
* (add) tika-eval/src/main/resources/common_tokens/nep
* (add) tika-eval/src/main/resources/common_tokens/ori
* (add) tika-eval/src/main/resources/common_tokens/por
* (add)
tika-eval/src/main/java/org/apache/tika/eval/tools/CommonTokenOverlapCounter.java
* (add) tika-eval/src/main/resources/common_tokens/fra
* (add) tika-eval/src/main/resources/common_tokens/sun
* (add) tika-eval/src/main/resources/common_tokens/mon
* (add) tika-eval/src/main/java/org/apache/tika/eval/tools/TrainTestSplit.java
* (add) tika-eval/src/main/resources/common_tokens/bel
* (add) tika-eval/src/main/resources/common_tokens/gle
* (add)
tika-eval/src/main/java/org/apache/tika/eval/langid/ProbingLanguageDetector.java
* (add) tika-eval/src/main/resources/common_tokens/div
* (add) tika-eval/src/main/resources/common_tokens/mhr
* (add) tika-eval/src/main/resources/common_tokens/ukr
* (delete) tika-eval/src/main/resources/common_tokens/it
* (add) tika-eval/src/main/resources/common_tokens/ell
* (add) tika-eval/src/main/resources/common_tokens/pol
* (delete) tika-eval/src/main/resources/common_tokens/vi
* (add) tika-eval/src/main/resources/common_tokens/tam
* (edit) tika-eval/src/test/java/org/apache/tika/eval/util/LanguageIdTest.java
* (add) tika-eval/src/main/resources/common_tokens/spa
* (add) tika-eval/src/main/java/org/apache/tika/eval/tools/LeipzigSampler.java
* (add) tika-eval/src/main/resources/common_tokens/glg
* (add) tika-eval/src/main/resources/common_tokens/kan
* (add) tika-eval/src/main/resources/common_tokens/jpn
* (add) tika-eval/src/main/resources/common_tokens/pan
* (add) tika-eval/src/main/resources/common_tokens/isl
* (delete) tika-eval/src/main/resources/common_tokens/id
* (add) tika-eval/src/main/resources/common_tokens/ast
* (add) tika-eval/src/main/resources/common_tokens/fas
* (add) tika-eval/src/main/resources/common_tokens/yid
* (delete) tika-eval/src/main/resources/common_tokens/hi
* (add) tika-eval/src/main/resources/common_tokens/hye
* (add) tika-eval/src/main/resources/common_tokens/swa
* (delete) tika-eval/src/main/resources/common_tokens/ja
* (delete) tika-eval/src/main/resources/common_tokens/bn
* (edit)
tika-eval/src/main/java/org/apache/tika/eval/tools/BatchTopCommonTokenCounter.java
* (delete) tika-eval/src/main/resources/common_tokens/zh-tw
* (add)
tika-eval/src/main/java/org/apache/tika/eval/langid/LanguageIDWrapper.java
* (add) tika-eval/src/main/resources/common_tokens/jav
* (add) tika-eval/src/main/resources/common_tokens/kat
* (add) tika-eval/src/main/resources/common_tokens/hin
* (add) tika-eval/src/main/resources/common_tokens/tel
* (add) tika-eval/src/main/resources/common_tokens/war
* (add) tika-eval/src/main/resources/common_tokens/eus
* (edit) tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
* (add) tika-eval/src/test/java/org/apache/tika/eval/langid/LangIdTest.java
* (add) tika-eval/src/main/resources/common_tokens/mal
* (edit)
tika-eval/src/main/java/org/apache/tika/eval/tools/TopCommonTokenCounter.java
* (add) tika-eval/src/main/resources/common_tokens/nno
* (add) tika-eval/src/main/resources/common_tokens/mlt
* (add) tika-eval/src/main/resources/common_tokens/ces
* (add) tika-eval/src/main/resources/common_tokens/lvs
* (add) tika-eval/src/main/resources/common_tokens/bak
* (edit)
tika-eval/src/main/java/org/apache/tika/eval/batch/EvalConsumersBuilder.java
* (add) tika-eval/src/main/resources/common_tokens/fao
* (delete) tika-eval/src/main/resources/common_tokens/fr
* (add) tika-eval/src/main/resources/common_tokens/azj
* (add) tika-eval/src/main/resources/common_tokens/mlg
* (add) tika-eval/src/main/resources/common_tokens/pes
* (delete) tika-eval/src/main/resources/common_tokens/ur
* (add) tika-eval/src/main/resources/common_tokens/slv
* (add) tika-eval/src/main/resources/common_tokens/ron
* (edit)
tika-eval/src/test/java/org/apache/tika/tools/TopCommonTokenCounterTest.java
* (add) tika-eval/src/main/resources/common_tokens/zul
* (add) tika-eval/src/main/resources/common_tokens/kin
* (add) tika-eval/src/main/resources/common_tokens/lat
* (add) tika-eval/src/main/resources/common_tokens/som
* (add) tika-eval/src/main/resources/common_tokens/guj
* (add) tika-eval/src/main/resources/common_tokens/mar
* (add) tika-eval/src/main/resources/common_tokens/tuk
* (add) tika-eval/src/main/resources/opennlp/model_20190626.bin
* (delete) tika-eval/src/main/resources/common_tokens/zh-cn
* (add)
tika-eval/src/main/java/org/apache/tika/eval/tokens/URLEmailNormalizingFilterFactory.java
* (add) tika-eval/src/main/resources/common_tokens/hrv
* (delete) tika-eval/src/main/resources/common_tokens/en
* (add) tika-eval/src/main/resources/common_tokens/tgl
* (add) tika-eval/src/main/resources/common_tokens/min
* (delete) tika-eval/src/main/resources/common_tokens/he
* (add) tika-eval/src/main/resources/common_tokens/tgk
* (add) tika-eval/src/main/resources/common_tokens/oci
* (add) tika-eval/src/main/resources/common_tokens/kor
* (add) tika-eval/src/main/resources/common_tokens/uig
* (add) tika-eval/src/main/resources/common_tokens/vol
* (add) tika-eval/src/main/resources/common_tokens/cym
* (delete) tika-eval/src/main/resources/common_tokens/fa
* (add) tika-eval/src/main/resources/common_tokens/lav
* (add) tika-eval/src/main/resources/common_tokens/ben
* (add) tika-eval/src/main/resources/common_tokens/ind
* (add) tika-eval/src/main/resources/common_tokens/sqi
* (add) tika-eval/src/main/resources/common_tokens/amh
* (add) tika-eval/src/main/resources/common_tokens/lim
* (add) tika-eval/src/main/resources/common_tokens/est
* (add) tika-eval/src/main/resources/common_tokens/uzb
* (add) tika-eval/src/main/resources/common_tokens/tat
* (add) tika-eval/src/main/resources/common_tokens/vie
* (add) tika-eval/src/main/resources/common_tokens/che
* (add) tika-eval/src/main/resources/common_tokens/plt
* (add) tika-eval/src/main/resources/common_tokens/eng
* (add) tika-eval/src/main/resources/common_tokens/bul
* (add) tika-eval/src/main/resources/common_tokens/ita
* (add) tika-eval/src/main/resources/common_tokens/tur
* (add) tika-eval/src/main/resources/common_tokens/urd
* (add) tika-eval/src/main/resources/common_tokens/ltz
* (edit) tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
* (add) tika-eval/src/main/resources/common_tokens/ban
* (delete) tika-eval/src/main/resources/common_tokens/nl
* (add) tika-eval/src/main/resources/common_tokens/tha
* (add) tika-eval/src/main/resources/common_tokens/lug
* (add) tika-eval/src/main/resources/common_tokens/fin
* (add) tika-eval/src/main/resources/common_tokens/nds
* (edit)
tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
* (edit)
tika-eval/src/main/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
* (edit)
tika-eval/src/main/java/org/apache/tika/eval/tools/SlowCompositeReaderWrapper.java
* (add) tika-eval/src/main/resources/common_tokens/xho
* (add) tika-eval/src/main/resources/common_tokens/kur
* (edit) tika-parsers/pom.xml
* (add) tika-eval/src/main/resources/common_tokens/heb
* (add) tika-eval/src/main/java/org/apache/tika/eval/langid/Language.java
* (add) tika-eval/src/main/resources/common_tokens/asm
* (add) tika-eval/src/main/resources/common_tokens/gsw
* (add) tika-eval/src/main/resources/common_tokens/kaz
* (add) tika-eval/src/main/resources/common_tokens/slk
* (add) tika-eval/src/main/resources/common_tokens/dan
* (delete) tika-eval/src/main/resources/common_tokens/es
* (delete)
tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
* (delete) tika-eval/src/main/resources/common_tokens/ar
* (add) tika-eval/src/main/resources/common_tokens/ara
* (add) tika-eval/src/main/resources/common_tokens/afr
* (add) tika-eval/src/main/resources/common_tokens/msa
* (add) tika-eval/src/main/resources/common_tokens/cmn
* (delete) tika-eval/src/main/resources/common_tokens/pt
* (edit)
tika-eval/src/test/java/org/apache/tika/eval/tokens/TokenCounterTest.java
* (add) tika-eval/src/main/resources/common_tokens/ceb
* (add) tika-eval/src/main/resources/common_tokens/ckb
* (add) tika-eval/src/main/resources/common_tokens/nan
* (edit) tika-eval/src/test/java/org/apache/tika/eval/AnalyzerManagerTest.java
* (add) tika-eval/src/main/resources/common_tokens/bre
* (add) tika-eval/src/main/resources/common_tokens/deu
* (add) tika-eval/src/main/resources/common_tokens/mri
* (edit) tika-eval/pom.xml
* (add) tika-eval/src/main/resources/common_tokens/hat
* (delete) tika-eval/src/main/resources/common_tokens/ko
* (add) tika-eval/src/main/resources/common_tokens/rus
* (add) tika-eval/src/main/java/org/apache/tika/eval/tools/LeipzigHelper.java
* (edit) tika-eval/src/main/resources/lucene-analyzers.json
* (edit)
tika-eval/src/main/java/org/apache/tika/eval/tokens/AlphaIdeographFilterFactory.java
* (add) tika-eval/src/main/resources/common_tokens/swe
* (add) tika-eval/src/main/resources/common_tokens/nob
* (add) tika-eval/src/main/resources/common_tokens/lit
* (add) tika-eval/src/main/resources/common_tokens/mkd
* (add) tika-eval/src/main/resources/common_tokens/pus
* (add) tika-eval/src/main/resources/common_tokens/san
* (delete) tika-eval/src/main/resources/common_tokens/de
* (delete) tika-eval/src/main/resources/common_tokens/el
> Consider switching lang-detection in tika-eval to open-nlp
> ----------------------------------------------------------
>
> Key: TIKA-2790
> URL: https://issues.apache.org/jira/browse/TIKA-2790
> Project: Tika
> Issue Type: Improvement
> Reporter: Tim Allison
> Priority: Major
> Attachments: fra_mixed_100000_0.0_0.txt, hasEnough.png,
> langid_20190509.zip, langid_20190510.zip, langid_20190514.zip,
> langid_20190514_plus_minus_1.zip, timeVsLength.png
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
