Revision: 8747
http://languagetool.svn.sourceforge.net/languagetool/?rev=8747&view=rev
Author: dominikoeo
Date: 2012-12-31 17:50:27 +0000 (Mon, 31 Dec 2012)
Log Message:
-----------
[fr] Switch to SRX sentence tokenizer in French.
Modified Paths:
--------------
trunk/JLanguageTool/CHANGES.txt
trunk/JLanguageTool/src/main/java/org/languagetool/language/French.java
trunk/JLanguageTool/src/main/resources/org/languagetool/resource/segment.srx
Modified: trunk/JLanguageTool/CHANGES.txt
===================================================================
--- trunk/JLanguageTool/CHANGES.txt 2012-12-31 15:02:33 UTC (rev 8746)
+++ trunk/JLanguageTool/CHANGES.txt 2012-12-31 17:50:27 UTC (rev 8747)
@@ -7,6 +7,9 @@
-Added about 9000 nameplaces to the dictionary. Source: enciclopedia.cat.
Some processing
done for generating (partial) gender and number data.
+ -French:
+ -SRX sentence tokenization
+
-bugfix: suggestions for compounds parts were missing sometimes
Modified:
trunk/JLanguageTool/src/main/java/org/languagetool/language/French.java
===================================================================
--- trunk/JLanguageTool/src/main/java/org/languagetool/language/French.java
2012-12-31 15:02:33 UTC (rev 8746)
+++ trunk/JLanguageTool/src/main/java/org/languagetool/language/French.java
2012-12-31 17:50:27 UTC (rev 8747)
@@ -1,6 +1,6 @@
-/* LanguageTool, a natural language style checker
+/* LanguageTool, a natural language style checker
* Copyright (C) 2007 Daniel Naber (http://www.danielnaber.de)
- *
+ *
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
@@ -32,6 +32,8 @@
import org.languagetool.rules.fr.QuestionWhitespaceRule;
import org.languagetool.rules.patterns.Unifier;
import org.languagetool.rules.spelling.hunspell.HunspellNoSuggestionRule;
+import org.languagetool.tokenizers.SRXSentenceTokenizer;
+import org.languagetool.tokenizers.SentenceTokenizer;
import org.languagetool.tagging.Tagger;
import org.languagetool.tagging.disambiguation.Disambiguator;
import org.languagetool.tagging.disambiguation.fr.FrenchHybridDisambiguator;
@@ -39,17 +41,26 @@
public class French extends Language {
+ private SentenceTokenizer sentenceTokenizer;
private Tagger tagger;
private Disambiguator disambiguator;
private Unifier unifier;
private Unifier disambiguationUnifier;
-
+
@Override
public Locale getLocale() {
return new Locale(getShortName());
}
@Override
+ public final SentenceTokenizer getSentenceTokenizer() {
+ if (sentenceTokenizer == null) {
+ sentenceTokenizer = new SRXSentenceTokenizer(this);
+ }
+ return sentenceTokenizer;
+ }
+
+ @Override
public String getName() {
return "French";
}
@@ -58,7 +69,7 @@
public String getShortName() {
return "fr";
}
-
+
@Override
public String[] getCountryVariants() {
return new String[]{"FR", "", "BE", "CH", "CA", "LU", "MC", "CM",
@@ -76,7 +87,7 @@
/*"»", French dialog can contain multiple sentences.
*/
/*"’" used in "d’arm" and many other words */ };
}
-
+
@Override
public Tagger getTagger() {
if (tagger == null) {
@@ -92,7 +103,7 @@
}
return disambiguator;
}
-
+
@Override
public Unifier getUnifier() {
if (unifier == null) {
Modified:
trunk/JLanguageTool/src/main/resources/org/languagetool/resource/segment.srx
===================================================================
---
trunk/JLanguageTool/src/main/resources/org/languagetool/resource/segment.srx
2012-12-31 15:02:33 UTC (rev 8746)
+++
trunk/JLanguageTool/src/main/resources/org/languagetool/resource/segment.srx
2012-12-31 17:50:27 UTC (rev 8747)
@@ -4704,6 +4704,30 @@
<afterbreak>\p{Lu}\p{Ll}</afterbreak>
</rule>
</languagerule>
+<languagerule languagerulename="French">
+<!-- French abbreviations -->
+<rule break="no">
+<beforebreak>\b(art|app|cf|chap|env|fém|fig|masc|p|sing|suiv|suppl|tél)\.\s</beforebreak>
+<afterbreak>\p{Ll}</afterbreak>
+</rule>
+<rule break="no">
+<beforebreak>\b(ave?|boul|Mr?|MM?)\.\s</beforebreak>
+<afterbreak></afterbreak>
+</rule>
+<!-- Break rules -->
+<rule break="yes">
+<beforebreak>[\.!?…][\u0002|'|"|«|\)|\]|\}]?\s+</beforebreak>
+<afterbreak></afterbreak>
+</rule>
+<rule break="yes">
+<beforebreak>[\.!?…]['"\p{Pe}\u00BB\u201D]?</beforebreak>
+<afterbreak>\p{Lu}[^\p{Lu}]</afterbreak>
+</rule>
+<rule break="yes">
+<beforebreak>\s\p{L}[\.!?…]\s</beforebreak>
+<afterbreak>\p{Lu}\p{Ll}</afterbreak>
+</rule>
+</languagerule>
<languagerule languagerulename="Ukrainian">
<rule break="no">
<beforebreak>\b\d+\.\s</beforebreak>
@@ -5097,6 +5121,7 @@
<languagemap languagepattern="(DE|de).*"
languagerulename="German"></languagemap>
<languagemap languagepattern="(DA|da).*"
languagerulename="Danish"></languagemap>
<languagemap languagepattern="(EO|eo).*"
languagerulename="Esperanto"></languagemap>
+<languagemap languagepattern="(FR|fr).*"
languagerulename="French"></languagemap>
<languagemap languagepattern="(UK|uk).*"
languagerulename="Ukrainian"></languagemap>
<languagemap languagepattern="(BE|be).*"
languagerulename="Belarusian"></languagemap>
<languagemap languagepattern="(GL|gl).*"
languagerulename="Galician"></languagemap>
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Master Visual Studio, SharePoint, SQL, ASP.NET, C# 2012, HTML5, CSS,
MVC, Windows 8 Apps, JavaScript and much more. Keep your skills current
with LearnDevNow - 3,200 step-by-step video tutorials by Microsoft
MVPs and experts. SALE $99.99 this month only -- learn more at:
http://p.sf.net/sfu/learnmore_122412
_______________________________________________
Languagetool-commits mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/languagetool-commits
