test-bugzilla-files/execute.sh | 2 test-bugzilla-files/new-control.py | 77 +++++++++++++++++++++-------- test-bugzilla-files/test-bugzilla-files.py | 22 +------- 3 files changed, 63 insertions(+), 38 deletions(-)
New commits: commit 77eb5fd822f0827e288e26940eb867858d6a6f7b Author: Markus Mohrhard <[email protected]> Date: Sat Nov 1 22:22:57 2014 +0100 move from dir based testing to task list based Instead of going through a whole directory in one task split the whole into 100 file tasks. This should make sure that all cores are always doing some work. diff --git a/test-bugzilla-files/execute.sh b/test-bugzilla-files/execute.sh index 9203a7e..e2c6076 100755 --- a/test-bugzilla-files/execute.sh +++ b/test-bugzilla-files/execute.sh @@ -1,4 +1,4 @@ mkdir /srv/crashtestdata/control/$1 cd /srv/crashtestdata/control/$1 -TMPDIR=/srv/crashtestdata/tmpdir /home/buildslave/build/instdir/program/python /home/buildslave/source/dev-tools/test-bugzilla-files/test-bugzilla-files.py --soffice=path:/home/buildslave/build/instdir/program/soffice --userdir=file:///home/buildslave/.config/libreoffice_$1/4 /home/buildslave/source/bugdocs3/$1 2>&1 | tee /srv/crashtestdata/console_$i.log +TMPDIR=/srv/crashtestdata/tmpdir /home/buildslave/build/instdir/program/python /home/buildslave/source/dev-tools/test-bugzilla-files/test-bugzilla-files.py --soffice=path:/home/buildslave/build/instdir/program/soffice --userdir=file:///home/buildslave/.config/libreoffice_$1/4 $1 2>&1 | tee /srv/crashtestdata/console_$i.log rm core* diff --git a/test-bugzilla-files/new-control.py b/test-bugzilla-files/new-control.py index 25617a7..c27cb5f 100644 --- a/test-bugzilla-files/new-control.py +++ b/test-bugzilla-files/new-control.py @@ -21,26 +21,63 @@ import os.path import concurrent.futures import time import subprocess +import getopt +import sys -def get_directories(): - d='.' - directories = [o for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))] - return directories +def partition(l, n): + for i in range(0, len(l), n): + yield l[i:i+n] + +def get_tasks(directory, files_per_task): + flist = [os.path.join(dirpath, f) for dirpath, dirnames, fnames in os.walk(directory) for f in fnames] + + partitioned_list = list(partition(flist, files_per_task)) + task_files = [] + i = 0 + for list_item in partitioned_list: + filename = "task" + str(i) + task_file = open(filename, "w") + for item in list_item: + task_file.write("%s\n" % item) + task_files.append(filename) + i += 1 + return task_files + +def execute_task(task_file): + # subprocess.call("./execute.sh " + task_file, shell=True) + time.sleep(1) + +def usage(): + message = """usage: {program} [option] dir" + - h | --help: print usage information + + 'dir' is the path to the directory with the test files""" + print(message.format(program = os.path.basename(sys.argv[0]))) + +if __name__ == "__main__": + opts, args = getopt.getopt(sys.argv[1:], "hd:", ["help", "directory="]) + print(args) + print(opts) + if "-h" in opts or "--help" in opts: + usage() + sys.exit() + + for opt, arg in opts: + if opt in ("-d", "--directory"): + directory = arg -def execute_task(directory): - print("Yeah") print(directory) - subprocess.call("./execute.sh " + directory, shell=True) - time.sleep(10) - return - -with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - future_to_task = {executor.submit(execute_task, dirs): dirs for dirs in get_directories()} - for future in concurrent.futures.as_completed(future_to_task): - task = future_to_task[future] - try: - future.result() - except Exception as exc: - print('%r generated an exception: %s' % (task, exc)) - else: - print('%r successfully passed' % (task)) + if not os.path.isdir(directory): + sys.exit(1) + + task_size = 100 + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + future_to_task = {executor.submit(execute_task, task_file): task_file for task_file in get_tasks(directory, task_size)} + for future in concurrent.futures.as_completed(future_to_task): + task = future_to_task[future] + try: + future.result() + except Exception as exc: + print('%r generated an exception: %s' % (task, exc)) + else: + print('%r successfully passed' % (task)) diff --git a/test-bugzilla-files/test-bugzilla-files.py b/test-bugzilla-files/test-bugzilla-files.py index 2bd907e..11cf41c 100644 --- a/test-bugzilla-files/test-bugzilla-files.py +++ b/test-bugzilla-files/test-bugzilla-files.py @@ -69,21 +69,9 @@ def partition(list, pred): right.append(e) return (left, right) -def filelist(dir, suffix): - if len(dir) == 0: - raise Exception("filelist: empty directory") - if not(dir[-1] == "/"): - dir += "/" - files = [dir + f for f in os.listdir(dir)] -# print(files) - return [f for f in files - if os.path.isfile(f) and os.path.splitext(f)[1] == suffix] - -def getFiles(dirs, suffix): -# print( dirs ) - files = [] - for dir in dirs: - files += filelist(dir, suffix) +def getFiles(file_list_name): + file_list = open(file_list_name, "r") + files = file_list.readlines() return files ### UNO utilities ### @@ -593,7 +581,7 @@ def parseArgs(argv): return (dict(optlist), args) def usage(): - message = """usage: {program} [option]... [directory]..." + message = """usage: {program} [option]... [task_file]..." -h | --help: print usage information --soffice=method:location specify soffice instance to connect to @@ -601,7 +589,7 @@ def usage(): --userdir=URL specify user installation directory for 'path' method --valgrind pass --valgrind to soffice for 'path' method - 'location' is a pathname, not a URL. 'userdir' is a URL. the 'directory' parameters should be + 'location' is a pathname, not a URL. 'userdir' is a URL. the 'task_file' parameters should be full absolute pathnames, not URLs.""" print(message.format(program = os.path.basename(sys.argv[0]))) _______________________________________________ Libreoffice-commits mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
