bin/find-german-comments | 23 ++++++++++++++--------- bin/text_cat/text_cat | 2 ++ 2 files changed, 16 insertions(+), 9 deletions(-)
New commits: commit 6efd9725912ae67bafbfe613220d9c09443ea0ba Author: Jeroen Nijhof <[email protected]> Date: Thu May 1 18:27:38 2014 +0200 Speed up find-german-comments Instead of recreating a text_cat sub-process for every string, start a single text-cat process (with the -s flag, "per line") and reuse that. Testing on svl/source shows a speed-up of 7.5 x, down from 45.6s to 6.1s! To avoid blocking on the pipes set autoflush in text-cat, and use read_line instead of read on the find-german-comments side. Change-Id: Ic6c00fc3c1e7cbcbbfff65512bf54ad11f900697 Reviewed-on: https://gerrit.libreoffice.org/9226 Reviewed-by: Michael Meeks <[email protected]> Tested-by: Michael Meeks <[email protected]> diff --git a/bin/find-german-comments b/bin/find-german-comments index 4910611..7c5f718 100755 --- a/bin/find-german-comments +++ b/bin/find-german-comments @@ -36,6 +36,7 @@ class Parser: """ def __init__(self): self.strip = string.punctuation + " \n" + self.text_cat = self.start_text_cat(); op = optparse.OptionParser() op.set_usage("%prog [options] <rootdir>\n\n" + "Searches for german comments in cxx/hxx source files inside a given root\n" + @@ -117,19 +118,23 @@ class Parser: yield (count, re.sub(".*/\*(.*)\*/.*", r"\1", i).strip(self.strip)) count += 1 - def get_lang(self, s): - """ the output is 'german' or 'english' or 'german or english'. when - unsure, just don't warn, there are strings where you just can't - teremine the results reliably, like '#110680#' """ + def start_text_cat(self): cwd = os.getcwd() # change to our directory os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0]) - sock = subprocess.Popen(["text_cat/text_cat", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - sock.stdin.write(s) - sock.stdin.close() - lang = sock.stdout.read().strip() - sock.stdout.close() + sock = subprocess.Popen(["text_cat/text_cat", "-s", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) os.chdir(cwd) + return sock + + def get_lang(self, s): + """ the output is 'german' or 'english' or 'german or english'. when + unsure, just don't warn, there are strings where you just can't + teremine the results reliably, like '#110680#' """ + + self.text_cat.stdin.write(s) + self.text_cat.stdin.write("\n") + self.text_cat.stdin.flush() + lang = self.text_cat.stdout.readline().strip() return lang def is_german(self, s): diff --git a/bin/text_cat/text_cat b/bin/text_cat/text_cat index 6c6b0d1..c907f50 100755 --- a/bin/text_cat/text_cat +++ b/bin/text_cat/text_cat @@ -19,6 +19,8 @@ $opt_f ||= 0; $opt_t ||= 400; $opt_u ||= 1.05; +$| = 1; # auto-flush stdout + sub help { print <<HELP Text Categorization. Typically used to determine the language of a _______________________________________________ Libreoffice-commits mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
