Revision: 7818 http://languagetool.svn.sourceforge.net/languagetool/?rev=7818&view=rev Author: dnaber Date: 2012-08-08 22:09:43 +0000 (Wed, 08 Aug 2012) Log Message: ----------- fix: use the right tokenizer for generating the ignore.txt file; remove date from ignore.txt as it confuses SVN (caused file to change although content is the same)
Modified Paths: -------------- trunk/JLanguageTool/src/java/org/languagetool/JLanguageTool.java trunk/JLanguageTool/src/java/org/languagetool/rules/spelling/SuggestionExtractor.java trunk/JLanguageTool/src/resource/ast/hunspell/ignore.txt trunk/JLanguageTool/src/resource/br/hunspell/ignore.txt trunk/JLanguageTool/src/resource/ca/hunspell/ignore.txt trunk/JLanguageTool/src/resource/da/hunspell/ignore.txt trunk/JLanguageTool/src/resource/de/hunspell/ignore.txt trunk/JLanguageTool/src/resource/el/hunspell/ignore.txt trunk/JLanguageTool/src/resource/en/hunspell/ignore.txt trunk/JLanguageTool/src/resource/eo/hunspell/ignore.txt trunk/JLanguageTool/src/resource/es/hunspell/ignore.txt trunk/JLanguageTool/src/resource/fr/hunspell/ignore.txt trunk/JLanguageTool/src/resource/gl/hunspell/ignore.txt trunk/JLanguageTool/src/resource/is/hunspell/ignore.txt trunk/JLanguageTool/src/resource/it/hunspell/ignore.txt trunk/JLanguageTool/src/resource/km/hunspell/ignore.txt trunk/JLanguageTool/src/resource/ml/hunspell/ignore.txt trunk/JLanguageTool/src/resource/nl/hunspell/ignore.txt trunk/JLanguageTool/src/resource/pl/hunspell/ignore.txt trunk/JLanguageTool/src/resource/pt/hunspell/ignore.txt trunk/JLanguageTool/src/resource/ro/hunspell/ignore.txt trunk/JLanguageTool/src/resource/ru/hunspell/ignore.txt trunk/JLanguageTool/src/resource/sk/hunspell/ignore.txt trunk/JLanguageTool/src/resource/sl/hunspell/ignore.txt trunk/JLanguageTool/src/resource/sv/hunspell/ignore.txt trunk/JLanguageTool/src/resource/tl/hunspell/ignore.txt trunk/JLanguageTool/src/resource/uk/hunspell/ignore.txt trunk/JLanguageTool/src/test/org/languagetool/rules/spelling/SuggestionExtractorTest.java Modified: trunk/JLanguageTool/src/java/org/languagetool/JLanguageTool.java =================================================================== --- trunk/JLanguageTool/src/java/org/languagetool/JLanguageTool.java 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/java/org/languagetool/JLanguageTool.java 2012-08-08 22:09:43 UTC (rev 7818) @@ -390,8 +390,8 @@ */ public void addRule(final Rule rule) { userRules.add(rule); - final SuggestionExtractor extractor = new SuggestionExtractor(language); - final List<String> suggestionTokens = extractor.getSuggestionTokens(rule); + final SuggestionExtractor extractor = new SuggestionExtractor(); + final List<String> suggestionTokens = extractor.getSuggestionTokens(rule, language); final List<Rule> allActiveRules = getAllActiveRules(); addIgnoreWords(suggestionTokens, allActiveRules); } @@ -433,8 +433,8 @@ final List<String> suggestionTokens = new ArrayList<String>(); for (Rule activeRule : allActiveRules) { if (activeRule instanceof PatternRule) { - final SuggestionExtractor extractor = new SuggestionExtractor(language); - suggestionTokens.addAll(extractor.getSuggestionTokens(activeRule)); + final SuggestionExtractor extractor = new SuggestionExtractor(); + suggestionTokens.addAll(extractor.getSuggestionTokens(activeRule, language)); } } return suggestionTokens; Modified: trunk/JLanguageTool/src/java/org/languagetool/rules/spelling/SuggestionExtractor.java =================================================================== --- trunk/JLanguageTool/src/java/org/languagetool/rules/spelling/SuggestionExtractor.java 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/java/org/languagetool/rules/spelling/SuggestionExtractor.java 2012-08-08 22:09:43 UTC (rev 7818) @@ -40,22 +40,19 @@ private final static Pattern SUGGESTION_PATTERN = Pattern.compile("<suggestion.*?>(.*?)</suggestion>"); private final static Pattern BACK_REFERENCE_PATTERN = Pattern.compile("\\\\" + "\\d+"); - private final Language language; - - public SuggestionExtractor(Language language) { - this.language = language; + public SuggestionExtractor() { } /** * Get the tokens of simple suggestions, i.e. those that don't use back references. */ - public List<String> getSuggestionTokens(Rule rule) { + public List<String> getSuggestionTokens(Rule rule, Language language) { final List<String> wordsToBeIgnored = new ArrayList<String>(); if (rule instanceof PatternRule) { final PatternRule patternRule = (PatternRule) rule; final String message = patternRule.getMessage(); final List<String> suggestions = getSimpleSuggestions(message); - final List<String> tokens = getSuggestionTokens(suggestions); + final List<String> tokens = getSuggestionTokens(suggestions, language); wordsToBeIgnored.addAll(tokens); } return wordsToBeIgnored; @@ -88,7 +85,7 @@ return !matcher.find(); } - private List<String> getSuggestionTokens(List<String> suggestions) { + private List<String> getSuggestionTokens(List<String> suggestions, Language language) { final List<String> tokens = new ArrayList<String>(); for (String suggestion : suggestions) { final List<String> suggestionTokens = language.getWordTokenizer().tokenize(suggestion); @@ -113,24 +110,22 @@ } final File ignoreFile = new File(hunspellDir, "ignore.txt"); final Set<String> tokens = entry.getValue(); - if (tokens.size() > 0) { - final FileWriter writer = new FileWriter(ignoreFile); - try { - writeIntro(writer, language); - for (String token : tokens) { - writer.write(token); - writer.write("\n"); - } - } finally { - writer.close(); + final FileWriter writer = new FileWriter(ignoreFile); + try { + writeIntro(writer, language); + for (String token : tokens) { + writer.write(token); + writer.write("\n"); } - System.out.println("Wrote " + tokens.size() + " words to " + ignoreFile); + } finally { + writer.close(); } + System.out.println("Wrote " + tokens.size() + " words to " + ignoreFile); } } private void writeIntro(FileWriter writer, Language language) throws IOException { - writer.write("# words to be ignored by the spellchecker (auto-generated " + new Date() + ")\n"); + writer.write("# words to be ignored by the spellchecker (auto-generated)\n"); writeArtificialTestCaseItems(writer, language); } @@ -160,7 +155,7 @@ int tokenCount = 0; int noErrorCount = 0; for (Rule rule : rules) { - final List<String> tokens = getSuggestionTokens(rule); + final List<String> tokens = getSuggestionTokens(rule, lang); tokenCount += tokens.size(); for (String token : tokens) { final AnalyzedSentence analyzedToken = languageTool.getAnalyzedSentence(token); @@ -208,7 +203,7 @@ } public static void main(String[] args) throws IOException { - final SuggestionExtractor extractor = new SuggestionExtractor(/*not used:*/Language.ENGLISH); + final SuggestionExtractor extractor = new SuggestionExtractor(); extractor.writeIgnoreTokensForLanguages(); } Modified: trunk/JLanguageTool/src/resource/ast/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/ast/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/ast/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:24 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/resource/br/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/br/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/br/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,9 +1,2 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) -hant -oc -hozh -hwi -hemãn -hichen -oac -hoc +# words to be ignored by the spellchecker (auto-generated) +o’ Modified: trunk/JLanguageTool/src/resource/ca/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/ca/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/ca/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,8 +1,5 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) -ls-hi -d +# words to be ignored by the spellchecker (auto-generated) ls -als -n -dels -pels +'ls +'n +d' Modified: trunk/JLanguageTool/src/resource/da/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/da/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/da/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:24 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/resource/de/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/de/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/de/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) einPseudoWortFürLanguageToolTests Unitymedia Open Modified: trunk/JLanguageTool/src/resource/el/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/el/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/el/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:25 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/resource/en/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/en/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/en/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) anArtificialTestWordForLanguageTool cui chock-full Modified: trunk/JLanguageTool/src/resource/eo/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/eo/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/eo/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) videkunveno telekunveno dank Modified: trunk/JLanguageTool/src/resource/es/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/es/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/es/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,2 +1,2 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) grosso Modified: trunk/JLanguageTool/src/resource/fr/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/fr/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/fr/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) feta baladodiffusée coconnage @@ -28,14 +28,12 @@ affecteur Meyer Garfunkel -mozarella annonciel Summer harceliciel Dead Lego mozarelle -équililbre aujourd Ozzy vidéolynchage @@ -56,7 +54,6 @@ Vicious Sid Streep -évue sexerie minilabo cybercarnet @@ -67,10 +64,8 @@ Britney -S Megadeth -fûté agaciel Edgar -gounelle bd cinédom boul @@ -86,6 +81,7 @@ Warren entreprenariat Mercury +loquette cyberpirate érothèque polluriel @@ -112,4 +108,3 @@ Cox ultrasecret Franzen -pr Modified: trunk/JLanguageTool/src/resource/gl/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/gl/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/gl/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) garavata pregravado parabenizar Modified: trunk/JLanguageTool/src/resource/is/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/is/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/is/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,2 +1,2 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) árdegisverður Modified: trunk/JLanguageTool/src/resource/it/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/it/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/it/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,2 +1,2 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) *** Modified: trunk/JLanguageTool/src/resource/km/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/km/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/km/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:25 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/resource/ml/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/ml/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/ml/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,9 +1,5 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) -ചങ്ങാതിമാര് +# words to be ignored by the spellchecker (auto-generated) മീരയെ രാജനോട് -പ്രവര് -ത്തകനേയും -അവള് -സംസ്കാരസമ്പന്നര് -പുഴയില് +സംസ്കാരസമ്പന്നര് +പ്രവര്ത്തകനേയും Modified: trunk/JLanguageTool/src/resource/nl/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/nl/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/nl/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) tempomaat Ameriknen botsballon Modified: trunk/JLanguageTool/src/resource/pl/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/pl/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/pl/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Katow spółdz vu Modified: trunk/JLanguageTool/src/resource/pt/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/pt/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/pt/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) ultra-secreto fim-de-semana pêlo Modified: trunk/JLanguageTool/src/resource/ro/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/ro/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/ro/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,2 +1,2 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) experință Modified: trunk/JLanguageTool/src/resource/ru/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/ru/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/ru/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:24 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/resource/sk/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/sk/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/sk/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:25 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/resource/sl/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/sl/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/sl/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) d št n Modified: trunk/JLanguageTool/src/resource/sv/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/sv/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/sv/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) fl acta notam Modified: trunk/JLanguageTool/src/resource/tl/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/tl/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/tl/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1,4 +1,4 @@ -# words to be ignored by the spellchecker (auto-generated Wed Aug 08 00:05:28 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) iskor riserts keyk Modified: trunk/JLanguageTool/src/resource/uk/hunspell/ignore.txt =================================================================== --- trunk/JLanguageTool/src/resource/uk/hunspell/ignore.txt 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/resource/uk/hunspell/ignore.txt 2012-08-08 22:09:43 UTC (rev 7818) @@ -1 +1 @@ -# words to be ignored by the spellchecker (auto-generated Tue Aug 07 22:48:25 CEST 2012) +# words to be ignored by the spellchecker (auto-generated) Modified: trunk/JLanguageTool/src/test/org/languagetool/rules/spelling/SuggestionExtractorTest.java =================================================================== --- trunk/JLanguageTool/src/test/org/languagetool/rules/spelling/SuggestionExtractorTest.java 2012-08-08 20:20:59 UTC (rev 7817) +++ trunk/JLanguageTool/src/test/org/languagetool/rules/spelling/SuggestionExtractorTest.java 2012-08-08 22:09:43 UTC (rev 7818) @@ -26,7 +26,7 @@ public class SuggestionExtractorTest extends TestCase { public void testGetSuggestions() throws IOException { - final SuggestionExtractor extractor = new SuggestionExtractor(Language.ENGLISH); + final SuggestionExtractor extractor = new SuggestionExtractor(); assertEquals("[]", extractor.getSimpleSuggestions("Did you mean foo?").toString()); assertEquals("[foo bla]", extractor.getSimpleSuggestions("Did you mean <suggestion>foo bla</suggestion>?").toString()); assertEquals("[foo bla, xxx]", extractor.getSimpleSuggestions("Did you mean <suggestion>foo bla</suggestion> or <suggestion>xxx</suggestion>?").toString()); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. ------------------------------------------------------------------------------ Live Security Virtual Conference Exclusive live event will cover all the ways today's security and threat landscape has changed and how IT managers can respond. Discussions will include endpoint security, mobile security and the latest in malware threats. http://www.accelacomm.com/jaw/sfrnl04242012/114/50122263/ _______________________________________________ Languagetool-cvs mailing list Languagetool-cvs@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/languagetool-cvs