Revision: 1350 Author: robhamerling Date: Sun Sep 27 01:27:22 2009 Log: Made check on mixed case for (pseudo) variables more general.
http://code.google.com/p/jallib/source/detail?r=1350 Modified: /trunk/tools/jallib.py ======================================= --- /trunk/tools/jallib.py Sun Sep 20 15:28:01 2009 +++ /trunk/tools/jallib.py Sun Sep 27 01:27:22 2009 @@ -5,15 +5,15 @@ # Author: Sebastien Lelong, Copyright (c) 2008, all rights reserved. # Adapted-by: # Compiler: -# +# # This file is part of jallib (http://jallib.googlecode.com) # Released under the BSD license (http://www.opensource.org/licenses/bsd-license.php) # # Sources: -# +# # Description: this script handles common tasks when using the jallib SVN repository # `jallib help` for more ! -# +# # Dependencies: according to the following actions, you'll need to install the # following libraries (if you don't use the action, you can skip it) # If you use the binary executables, you won't need to install them @@ -24,7 +24,7 @@ # Of course, since it's about jal, you'll need jalv2 compiler installed # (not bundled in binary executables) # - jalv2 executable (compile): http://www.casadeyork.com/jalv2 -# +# # Notes: # @@ -36,10 +36,10 @@ import types as pytypes try: - import Cheetah.Template - has_cheetah = True + import Cheetah.Template + has_cheetah = True except ImportError: - has_cheetah = False + has_cheetah = False try: import subprocess @@ -48,10 +48,10 @@ has_subprocess = False try: - import picshell.engine.core.PicEngine - has_picshell = True + import picshell.engine.core.PicEngine + has_picshell = True except ImportError: - has_picshell = False + has_picshell = False ######### @@ -59,50 +59,50 @@ ######### def get_jal_filenames(dir,subdir=None,predicate=lambda dir,filename: True): - jalfiles = {} - if os.path.isfile(dir): - return {dir:dir} - for df in os.listdir(dir): - if df.startswith("."): - continue - fulldf = os.path.join(dir,df) - halfdf = os.path.join(subdir and subdir or "",df) - if os.path.isfile(fulldf) and fulldf.endswith(".jal") and predicate(dir,df): - jalfiles[df] = halfdf - elif os.path.isdir(fulldf): - jalfiles.update(get_jal_filenames(fulldf,subdir=os.path.join(subdir and subdir or "",df),predicate=predicate)) - return jalfiles + jalfiles = {} + if os.path.isfile(dir): + return {dir:dir} + for df in os.listdir(dir): + if df.startswith("."): + continue + fulldf = os.path.join(dir,df) + halfdf = os.path.join(subdir and subdir or "",df) + if os.path.isfile(fulldf) and fulldf.endswith(".jal") and predicate(dir,df): + jalfiles[df] = halfdf + elif os.path.isdir(fulldf): + jalfiles.update(get_jal_filenames(fulldf,subdir=os.path.join(subdir and subdir or "",df),predicate=predicate)) + return jalfiles def get_full_sample_path(sample_dir,pic="",sample=""): - return os.path.join(sample_dir,sample) + return os.path.join(sample_dir,sample) # cache found files explored_tests = None def get_full_test_path(sample_dir,test=""): - global explored_tests - testdir = os.path.join(sample_dir,"..","test") - if not test: - return testdir - else: - if not explored_tests: - # exclude board - tdirs = [(d,os.path.join(testdir,d)) for d in os.listdir(testdir) if not d.startswith(".") and d != "board"] - # search for test file... - test_files = {} - for (subdir,tdir) in tdirs: - test_files.update(get_jal_filenames(tdir,subdir)) ## if f.startswith("test_")]) - explored_tests = test_files - - return os.path.join(testdir,explored_tests[test]) + global explored_tests + testdir = os.path.join(sample_dir,"..","test") + if not test: + return testdir + else: + if not explored_tests: + # exclude board + tdirs = [(d,os.path.join(testdir,d)) for d in os.listdir(testdir) if not d.startswith(".") and d != "board"] + # search for test file... + test_files = {} + for (subdir,tdir) in tdirs: + test_files.update(get_jal_filenames(tdir,subdir)) ## if f.startswith("test_")]) + explored_tests = test_files + + return os.path.join(testdir,explored_tests[test]) def get_full_board_path(sample_dir,board=""): - return os.path.join(sample_dir,"..","test","board",board) - + return os.path.join(sample_dir,"..","test","board",board) + def find_includes(jalfile): - content = file(jalfile).read() - return re.findall("^\s*include\s+(\w+)\s*",content,re.MULTILINE) + content = file(jalfile).read() + return re.findall("^\s*include\s+(\w+)\s*",content,re.MULTILINE) ################ @@ -115,114 +115,114 @@ #---------# def _explore_dir(onedir): - dirs = [onedir] - def keepit(fd): - if os.path.isdir(fd): - dirs.extend(_explore_dir(fd)) - for fd in os.listdir(onedir): - if not fd.startswith("."): - keepit(os.path.join(onedir,fd)) - return dirs + dirs = [onedir] + def keepit(fd): + if os.path.isdir(fd): + dirs.extend(_explore_dir(fd)) + for fd in os.listdir(onedir): + if not fd.startswith("."): + keepit(os.path.join(onedir,fd)) + return dirs def clean_compiler_products(fromjalfile): - outdir = os.path.dirname(fromjalfile) or os.path.curdir - outfile = os.path.basename(fromjalfile) - noext = outfile[:-4] - for f in os.listdir(outdir): - # luckily all product files have 3-letters extension - if f[:-4] == noext and f[-4:] in [".asm",".hex",".err",".cod",".obj",".lst"]: - toclean = os.path.join(outdir,f) - ###print >> sys.stderr, "Cleaning %s" % toclean - try: - os.unlink(toclean) - except OSError,e: - print >> sys.stderr, "Can't clean %s because: %s" % (toclean,e) + outdir = os.path.dirname(fromjalfile) or os.path.curdir + outfile = os.path.basename(fromjalfile) + noext = outfile[:-4] + for f in os.listdir(outdir): + # luckily all product files have 3-letters extension + if f[:-4] == noext and f[-4:] in [".asm",".hex",".err",".cod",".obj",".lst"]: + toclean = os.path.join(outdir,f) + ###print >> sys.stderr, "Cleaning %s" % toclean + try: + os.unlink(toclean) + except OSError,e: + print >> sys.stderr, "Can't clean %s because: %s" % (toclean,e) def do_compile(args,exitonerror=True,clean=False,stdout=None,stderr=None): - if not has_subprocess: - print >> sys.stderr, "You can't use this action, because subprocess module is not installed" - sys.exit(255) - - # If we can't parse given args, this means there're all - # for the compiler. Keep them as backup - original_args = args - - # Only parse jallib arg, leave other for underlying cmd - def parse_args(args,limit=0): - if limit: - args = args[:limit] - if not args: - return (),() - # If error, eats args from the end, until no error: seperate args for - # jallib and args for compiler (not that clean...) - try: - opts, args = getopt.getopt(args, ACTIONS['compile']['options']) - return opts,args - except getopt.error,e: - return parse_args(args,-1) - - # There's not even an argument for the compiler... - if not original_args: - print >> sys.stderr, "Missing arguments to jalv2 compiler..." - sys.exit(255) - - # Try to extract jallib args - opts, args = parse_args(args) - if not opts: - # If it gives nothing, this means all given args - # are for the compiler. Restore original args. - args = original_args - else: - # there are opts for jallib, but args for jalv2 may have been - # eaten in parse_args(). Need to extract them according to opts - jallib_args = [] - map(lambda i: jallib_args.extend(i),opts) - args = list(set(original_args).difference(set(jallib_args))) - - - jalv2_exec = None - dirs = None - for o,v in opts: - if o == '-R': - dirs = [] - gdirs = v.split(";") - for d in gdirs: - dirs.extend(_explore_dir(d)) - elif o == '-E': - jalv2_exec = v - else: - print >> sys.stderr, "Wrong option %s" % o - # No root specified ? Try env var, else defaut to cwd - if not dirs: - v = os.environ.get('JALLIB_REPOS',os.path.curdir) - gdirs = v.split(";") - for d in gdirs: - dirs = _explore_dir(d) - # No exec specify, try env var or defaulting - if not jalv2_exec: - jalv2_exec = os.environ.get('JALLIB_JALV2','jalv2').split() - cmd = jalv2_exec + ["-s",";".join(dirs)] - # Complete with compiler args - if args: - cmd.extend(args) - try: - status = subprocess.check_call(cmd,shell=False,stdout=stdout,stderr=stderr) - if clean: - # assume srcfile is last arg (it not, can't clean) - srcfile = cmd[-1] - if not srcfile.endswith(".jal"): - print >> sys.stderr, "Can't clean, because can't know which file is the source file" - return status - clean_compiler_products(srcfile) - - return status - - except subprocess.CalledProcessError,e: - print >> sys.stderr, "Error while compiling file (status=%s).\nSee previous message." % e.returncode - exitonerror and sys.exit(e.returncode) - return e.returncode + if not has_subprocess: + print >> sys.stderr, "You can't use this action, because subprocess module is not installed" + sys.exit(255) + + # If we can't parse given args, this means there're all + # for the compiler. Keep them as backup + original_args = args + + # Only parse jallib arg, leave other for underlying cmd + def parse_args(args,limit=0): + if limit: + args = args[:limit] + if not args: + return (),() + # If error, eats args from the end, until no error: seperate args for + # jallib and args for compiler (not that clean...) + try: + opts, args = getopt.getopt(args, ACTIONS['compile']['options']) + return opts,args + except getopt.error,e: + return parse_args(args,-1) + + # There's not even an argument for the compiler... + if not original_args: + print >> sys.stderr, "Missing arguments to jalv2 compiler..." + sys.exit(255) + + # Try to extract jallib args + opts, args = parse_args(args) + if not opts: + # If it gives nothing, this means all given args + # are for the compiler. Restore original args. + args = original_args + else: + # there are opts for jallib, but args for jalv2 may have been + # eaten in parse_args(). Need to extract them according to opts + jallib_args = [] + map(lambda i: jallib_args.extend(i),opts) + args = list(set(original_args).difference(set(jallib_args))) + + + jalv2_exec = None + dirs = None + for o,v in opts: + if o == '-R': + dirs = [] + gdirs = v.split(";") + for d in gdirs: + dirs.extend(_explore_dir(d)) + elif o == '-E': + jalv2_exec = v + else: + print >> sys.stderr, "Wrong option %s" % o + # No root specified ? Try env var, else defaut to cwd + if not dirs: + v = os.environ.get('JALLIB_REPOS',os.path.curdir) + gdirs = v.split(";") + for d in gdirs: + dirs = _explore_dir(d) + # No exec specify, try env var or defaulting + if not jalv2_exec: + jalv2_exec = os.environ.get('JALLIB_JALV2','jalv2').split() + cmd = jalv2_exec + ["-s",";".join(dirs)] + # Complete with compiler args + if args: + cmd.extend(args) + try: + status = subprocess.check_call(cmd,shell=False,stdout=stdout,stderr=stderr) + if clean: + # assume srcfile is last arg (it not, can't clean) + srcfile = cmd[-1] + if not srcfile.endswith(".jal"): + print >> sys.stderr, "Can't clean, because can't know which file is the source file" + return status + clean_compiler_products(srcfile) + + return status + + except subprocess.CalledProcessError,e: + print >> sys.stderr, "Error while compiling file (status=%s).\nSee previous message." % e.returncode + exitonerror and sys.exit(e.returncode) + return e.returncode @@ -231,231 +231,231 @@ #----------# def content_not_empty(val): - return val.strip() != '' + return val.strip() != '' def compiler_version(val): - return re.match("^(>|<|>=|<=|=)?\d+(\.\d+\w*)+\s+$",val) + return re.match("^(>|<|>=|<=|=)?\d+(\.\d+\w*)+\s+$",val) JALLIB = """^-- This file is part of jallib\s+\(http://jallib.googlecode.com\)""" LICENSE = """^-- Released under the BSD license\s+\(http://www.opensource.org/licenses/bsd-license.php\)""" LICENSE2 = """^-- Released under the ZLIB license\s+\(http://www.opensource.org/licenses/zlib-license.html\)""" # exceptions while checking case -ALLOWED_MIXED_CASE = """.*(pin|port|tris|option|ancon).*""" +ALLOWED_MIXED_CASE = """.*(pin|port|_shadow|_flush).*""" FIELDS = [ - {'field':"Title",'predicate' : content_not_empty,'mandatory': True, 'multiline' :False}, - {'field':"Author",'predicate':content_not_empty,'mandatory':True,'multiline':False}, - {'field':"Adapted-by",'predicate':content_not_empty,'mandatory':False,'multiline':False}, - {'field':"Compiler",'predicate':compiler_version,'mandatory':True,'multiline':False}, - {'field':"Description",'predicate':content_not_empty,'mandatory':True,'multiline':True}, - {'field':"Sources",'predicate':content_not_empty,'mandatory':False,'multiline':True}, - {'field':"Notes",'predicate':content_not_empty,'mandatory':False,'multiline':True}, - ] + {'field':"Title",'predicate' : content_not_empty,'mandatory': True, 'multiline' :False}, + {'field':"Author",'predicate':content_not_empty,'mandatory':True,'multiline':False}, + {'field':"Adapted-by",'predicate':content_not_empty,'mandatory':False,'multiline':False}, + {'field':"Compiler",'predicate':compiler_version,'mandatory':True,'multiline':False}, + {'field':"Description",'predicate':content_not_empty,'mandatory':True,'multiline':True}, + {'field':"Sources",'predicate':content_not_empty,'mandatory':False,'multiline':True}, + {'field':"Notes",'predicate':content_not_empty,'mandatory':False,'multiline':True}, + ] errors = [] warnings = [] def extract_header(content): - # reverse() to pop header lines (no such shift()) - content.reverse() - header = [] - while True: - i,l = content.pop() - if not l.startswith("--"): - # not a header line, put it again - content.append((i,l)) - break - else: - header.append((i,l)) - # remaining content: back to forward... - content.reverse() - return header + # reverse() to pop header lines (no such shift()) + content.reverse() + header = [] + while True: + i,l = content.pop() + if not l.startswith("--"): + # not a header line, put it again + content.append((i,l)) + break + else: + header.append((i,l)) + # remaining content: back to forward... + content.reverse() + return header def validate_field(data,field,predicate,mandatory,multiline=False): - syntax = "^-- %s:\s*(.*)" % field - - def single_line_content(line): - return "".join(re.sub(syntax,lambda x:x.group(1),line)) - def multi_line_content(num,line,data): - c = single_line_content(line) - for i,l in data[data.index((num,line)) + 1:]: - # Check no comment gap - if not re.match("^--\s",l): - errors.append("%d: cannot extract content, line is not starting with comment: %s" % (i,repr(l))) - return "" - # No other field within content - for f in [d['field'] for d in FIELDS]: - if l.startswith("-- %s:" % f): - errors.append("%d: detected field %s within content of field %s" % (i,f,field)) - return "" - if l.strip() == '--': - # got end of block - break - c += re.sub("^--\s","",l) - else: - errors.append("Cannot find end of field content %s" % field) - return c - - # eat header, and check fields (mandatory or not) and - # content (using predicate) - c = None - for i,l in data: - if re.match(syntax,l): - # got it, check field content - if multiline: - c = multi_line_content(i,l,data) - else: - c = single_line_content(l) - # and check with predicate - valid = predicate(c) - if mandatory and not valid: - errors.append("%d: content for field %s is not valid: %s" % (i,field,repr(c))) - break - else: - if mandatory: - errors.append("Cannot find field %s (searched for '%s')" % (field,syntax)) - - return c + syntax = "^-- %s:\s*(.*)" % field + + def single_line_content(line): + return "".join(re.sub(syntax,lambda x:x.group(1),line)) + def multi_line_content(num,line,data): + c = single_line_content(line) + for i,l in data[data.index((num,line)) + 1:]: + # Check no comment gap + if not re.match("^--\s",l): + errors.append("%d: cannot extract content, line is not starting with comment: %s" % (i,repr(l))) + return "" + # No other field within content + for f in [d['field'] for d in FIELDS]: + if l.startswith("-- %s:" % f): + errors.append("%d: detected field %s within content of field %s" % (i,f,field)) + return "" + if l.strip() == '--': + # got end of block + break + c += re.sub("^--\s","",l) + else: + errors.append("Cannot find end of field content %s" % field) + return c + + # eat header, and check fields (mandatory or not) and + # content (using predicate) + c = None + for i,l in data: + if re.match(syntax,l): + # got it, check field content + if multiline: + c = multi_line_content(i,l,data) + else: + c = single_line_content(l) + # and check with predicate + valid = predicate(c) + if mandatory and not valid: + errors.append("%d: content for field %s is not valid: %s" % (i,field,repr(c))) + break + else: + if mandatory: + errors.append("Cannot find field %s (searched for '%s')" % (field,syntax)) + + return c def validate_header(content): - header = extract_header(content) - - # check stuff about jallib and license - jallib = False - license = False - for i,line in header: - if re.match(JALLIB,line): - jallib = True - if license: break - if re.match(LICENSE,line) or re.match(LICENSE2,line): - license = True - if jallib: break - not jallib and errors.append("Cannot find references to jallib (should have: %s)" % repr(JALLIB)) - not license and errors.append("Cannot find references to license") - for field_dict in FIELDS: - validate_field(header,**field_dict) + header = extract_header(content) + + # check stuff about jallib and license + jallib = False + license = False + for i,line in header: + if re.match(JALLIB,line): + jallib = True + if license: break + if re.match(LICENSE,line) or re.match(LICENSE2,line): + license = True + if jallib: break + not jallib and errors.append("Cannot find references to jallib (should have: %s)" % repr(JALLIB)) + not license and errors.append("Cannot find references to license") + for field_dict in FIELDS: + validate_field(header,**field_dict) def validate_filename(filename): - # must be lowercase - filename = os.path.basename(filename) - if filename.lower() != filename: - errors.append("Filename is not lowercase: %s" % filename) - # should have "jal" extention - if not filename.endswith(".jal"): - warnings.append("Filename doesn't have '*.jal' extention: %s" % filename) - pass + # must be lowercase + filename = os.path.basename(filename) + if filename.lower() != filename: + errors.append("Filename is not lowercase: %s" % filename) + # should have "jal" extention + if not filename.endswith(".jal"): + warnings.append("Filename doesn't have '*.jal' extention: %s" % filename) + pass def validate_lower_case(content): - # tokenize content, searching for infamous CamelCase words - # Also search Capitalized ones... - tokenizer = re.compile("\w+") - caps = re.compile("[A-Z]") - freetext = re.compile('(("|\').*("|\'))') - mixed = re.compile(ALLOWED_MIXED_CASE,re.IGNORECASE) - # no check in comments - codeonly = [] - for i,line in content: - if line.startswith(";") or line.startswith("--"): - continue - # don't consider free text (between quotes) - line = freetext.sub("",line) - l = line.split() - def nocomment(l,c): - # dirty search comments... - if c in l: - l = l[:l.index(c)] - tmp = [w.startswith(c) for n,w in enumerate(l)] - if True in tmp: - l = l[:tmp.index(True)] - return l - # first check ";" *then* "--" - # so things like ";;;;-- Note" are not considered - # (else produces wrong splitting...) - l = nocomment(l,";") - l = nocomment(l,"--") - # propagate line number - codeonly.append((i," ".join(l))) - - weird = {} # stores all errors - for i,line in codeonly: - for token in tokenizer.findall(line): - # it can be all in caps (constants) - if token.upper() == token: - continue - # hex definition, give up - if "0x" in token: - continue - # exceptions... - if mixed.match(token.lower()): - continue - if caps.findall(token): - weird.setdefault("Found Capitals in token: %s" % repr(token),[]).append(i) - # reconsitute errors, with multiple line number if error has occured multiple time - for s,nums in weird.items(): - err = "%s: %s" % (",".join(map(str,nums)),s) - errors.append(err) + # tokenize content, searching for infamous CamelCase words + # Also search Capitalized ones... + tokenizer = re.compile("\w+") + caps = re.compile("[A-Z]") + freetext = re.compile('(("|\').*("|\'))') + mixed = re.compile(ALLOWED_MIXED_CASE,re.IGNORECASE) + # no check in comments + codeonly = [] + for i,line in content: + if line.startswith(";") or line.startswith("--"): + continue + # don't consider free text (between quotes) + line = freetext.sub("",line) + l = line.split() + def nocomment(l,c): + # dirty search comments... + if c in l: + l = l[:l.index(c)] + tmp = [w.startswith(c) for n,w in enumerate(l)] + if True in tmp: + l = l[:tmp.index(True)] + return l + # first check ";" *then* "--" + # so things like ";;;;-- Note" are not considered + # (else produces wrong splitting...) + l = nocomment(l,";") + l = nocomment(l,"--") + # propagate line number + codeonly.append((i," ".join(l))) + + weird = {} # stores all errors + for i,line in codeonly: + for token in tokenizer.findall(line): + # it can be all in caps (constants) + if token.upper() == token: + continue + # hex definition, give up + if "0x" in token: + continue + # exceptions... + if mixed.match(token.lower()): + continue + if caps.findall(token): + weird.setdefault("Found Capitals in token: %s" % repr(token),[]).append(i) + # reconsitute errors, with multiple line number if error has occured multiple time + for s,nums in weird.items(): + err = "%s: %s" % (",".join(map(str,nums)),s) + errors.append(err) def validate_procfunc_defs(content): - # no () in definition - func_proc = re.compile("^(procedure|function)") - no_spaces = re.compile(".*\s+\(.*is") - for i,line in content: - # don't even check both (), not needed - if func_proc.match(line) and not "(" in line: - errors.append("%d: %s missing (). Calls must also be explicit" % (i,repr(line))) - if func_proc.match(line) and no_spaces.match(line): - errors.append("%d: found a space before parenthesis: %s" % (i,repr(line))) + # no () in definition + func_proc = re.compile("^(procedure|function)") + no_spaces = re.compile(".*\s+\(.*is") + for i,line in content: + # don't even check both (), not needed + if func_proc.match(line) and not "(" in line: + errors.append("%d: %s missing (). Calls must also be explicit" % (i,repr(line))) + if func_proc.match(line) and no_spaces.match(line): + errors.append("%d: found a space before parenthesis: %s" % (i,repr(line))) def validate_code(content): - validate_lower_case(content) - validate_procfunc_defs(content) - # ... + validate_lower_case(content) + validate_procfunc_defs(content) + # ... def validate(filename): - validate_filename(filename) - # also extract line number (enumerate from 0, count from 1) - content = [(i + 1,l) for i,l in enumerate(open(filename,"r").readlines())] - validate_header(content) - # remaining content has no more header - validate_code(content) + validate_filename(filename) + # also extract line number (enumerate from 0, count from 1) + content = [(i + 1,l) for i,l in enumerate(open(filename,"r").readlines())] + validate_header(content) + # remaining content has no more header + validate_code(content) def report(filename): - print >> sys.stderr, "File: %s" % filename - print >> sys.stderr, "%d errors found" % len(errors) - for err in errors: - print >> sys.stderr, "\tERROR: %s" % err - print >> sys.stderr - print >> sys.stderr, "%d warnings found" % len(warnings) - for warn in warnings: - print >> sys.stderr, "\twarning: %s" % warn - - if errors or warnings: - return True + print >> sys.stderr, "File: %s" % filename + print >> sys.stderr, "%d errors found" % len(errors) + for err in errors: + print >> sys.stderr, "\tERROR: %s" % err + print >> sys.stderr + print >> sys.stderr, "%d warnings found" % len(warnings) + for warn in warnings: + print >> sys.stderr, "\twarning: %s" % warn + + if errors or warnings: + return True def do_validate(args): - # No jallib args (yet!) for validating - # args contain jal file to validate - - global errors - global warnings - at_least_one_failed = False - for filename in args: - validate(filename) - if report(filename): - at_least_one_failed = True - errors = [] - warnings = [] - - if at_least_one_failed: - sys.exit(1) - else: - sys.exit(0) + # No jallib args (yet!) for validating + # args contain jal file to validate + + global errors + global warnings + at_least_one_failed = False + for filename in args: + validate(filename) + if report(filename): + at_least_one_failed = True + errors = [] + warnings = [] + + if at_least_one_failed: + sys.exit(1) + else: + sys.exit(0) #--------# @@ -463,178 +463,178 @@ #--------# def parse_tags(content,*args): - current_tag = None - current_value = None - restags = {} - for l in content: - if "@jallib" in l: - what = l.split() - # only keep "interesting" jallib invokations - if len(what) == 3: - atjallib,tag,value = l.split() - if tag in args: - current_tag = tag - current_value = value - restags.setdefault(current_tag,{})[current_value] = [] - - if current_tag: - restags[current_tag][current_value].append(l) - return restags - + current_tag = None + current_value = None + restags = {} + for l in content: + if "@jallib" in l: + what = l.split() + # only keep "interesting" jallib invokations + if len(what) == 3: + atjallib,tag,value = l.split() + if tag in args: + current_tag = tag + current_value = value + restags.setdefault(current_tag,{})[current_value] = [] + + if current_tag: + restags[current_tag][current_value].append(l) + return restags + def parse_sections(content): - return parse_tags(content,"section") + return parse_tags(content,"section") def merge_board_testfile(boardcontent,testcontent): - board = parse_sections(boardcontent) - # replace sections in testcontent - testcontent = os.linesep.join(testcontent) - toreplace = [m for m in re.finditer("((--)+)|(;+)\...@jallib use (.*)",testcontent,re.MULTILINE) if m.groups()[-1]] - newcontent = "" - start = 0 - for m in toreplace: - sectionname = m.groups()[-1].strip() - # when eating line sep char (to keep layout), - # remember some OS needs 2 chars ! - newcontent += testcontent[start:m.start() - len(os.linesep)] - new = os.linesep.join(board['section'][sectionname]) - start = m.end() + 1 # next char - newcontent += new - newcontent += testcontent[start:] - return newcontent + board = parse_sections(boardcontent) + # replace sections in testcontent + testcontent = os.linesep.join(testcontent) + toreplace = [m for m in re.finditer("((--)+)|(;+)\...@jallib use (.*)",testcontent,re.MULTILINE) if m.groups()[-1]] + newcontent = "" + start = 0 + for m in toreplace: + sectionname = m.groups()[-1].strip() + # when eating line sep char (to keep layout), + # remember some OS needs 2 chars ! + newcontent += testcontent[start:m.start() - len(os.linesep)] + new = os.linesep.join(board['section'][sectionname]) + start = m.end() + 1 # next char + newcontent += new + newcontent += testcontent[start:] + return newcontent def normalize_linefeed(content): - # use single char - content = content.replace("\r\n","\n") - # split and join using OS settings - lines = re.split("[\n|\r]",content,re.MULTILINE) - content = "\n".join(lines) - return content + # use single char + content = content.replace("\r\n","\n") + # split and join using OS settings + lines = re.split("[\n|\r]",content,re.MULTILINE) + content = "\n".join(lines) + return content def generate_one_sample(boardfile,testfile,outfile,deleteiffailed=True): - # try to find which linefeed is used - board = file(boardfile).read().splitlines() - test = file(testfile).read().splitlines() - # keep test's headers, but enrich them with info about how files were merged - # headers need index (enumerate() on content) - # extract_header will change content in place, ie. will remove - # header from test content. - test = [t for t in enumerate(test)] - header = extract_header(test) - header = os.linesep.join([h for i,h in header]) - header += os.linesep.join([ - os.linesep + "--", - "-- This file has been generated from:", - "-- * board: %s" % os.path.basename(boardfile), - "-- * test : %s" % os.path.basename(testfile), - "--", - os.linesep]) - # back to content without index - test = [l for i,l in test] - merged = merge_board_testfile(board,test) - # wb: write binary format, no ASCII/chars interpretation - fout = file(outfile,"wb") - fout.write(header) - fout.write(merged) - fout.close() - - # compile it ! - status = do_compile([outfile],exitonerror=False,clean=True) - if status == 0: - print "Succesfully generated sample '%s' from board '%s' and test '%s'" % (outfile,boardfile,testfile) - elif deleteiffailed: - # delete the file ! - os.unlink(outfile) - raise Exception("Can't compile sample '%s' generated from '%s' and test '%s'" % (outfile,boardfile,testfile)) - + # try to find which linefeed is used + board = file(boardfile).read().splitlines() + test = file(testfile).read().splitlines() + # keep test's headers, but enrich them with info about how files were merged + # headers need index (enumerate() on content) + # extract_header will change content in place, ie. will remove + # header from test content. + test = [t for t in enumerate(test)] + header = extract_header(test) + header = os.linesep.join([h for i,h in header]) + header += os.linesep.join([ + os.linesep + "--", + "-- This file has been generated from:", + "-- * board: %s" % os.path.basename(boardfile), + "-- * test : %s" % os.path.basename(testfile), + "--", + os.linesep]) + # back to content without index + test = [l for i,l in test] + merged = merge_board_testfile(board,test) + # wb: write binary format, no ASCII/chars interpretation + fout = file(outfile,"wb") + fout.write(header) + fout.write(merged) + fout.close() + + # compile it ! + status = do_compile([outfile],exitonerror=False,clean=True) + if status == 0: + print "Succesfully generated sample '%s' from board '%s' and test '%s'" % (outfile,boardfile,testfile) + elif deleteiffailed: + # delete the file ! + os.unlink(outfile) + raise Exception("Can't compile sample '%s' generated from '%s' and test '%s'" % (outfile,boardfile,testfile)) + def find_board_files(boarddir): - return [os.path.join(boarddir,f) for f in os.listdir(boarddir) if not f.startswith(".") and f.startswith("board_") and f.endswith(".jal")] + return [os.path.join(boarddir,f) for f in os.listdir(boarddir) if not f.startswith(".") and f.startswith("board_") and f.endswith(".jal")] def find_test_files(testdir): - # testdir contains "board" dir, to be excluded - testfiles = [] - for d in [d for d in os.listdir(testdir) if d != "board" and not d.startswith(".")]: - d = os.path.join(testdir,d) - testfiles.extend([os.path.join(d,v) for v in get_jal_filenames(d,predicate=lambda _,x: x.startswith("test_")).values()]) - return testfiles + # testdir contains "board" dir, to be excluded + testfiles = [] + for d in [d for d in os.listdir(testdir) if d != "board" and not d.startswith(".")]: + d = os.path.join(testdir,d) + testfiles.extend([os.path.join(d,v) for v in get_jal_filenames(d,predicate=lambda _,x: x.startswith("test_")).values()]) + return testfiles def preferred_board(board): - data = file(board).read() - # well, should not consider comment but... should not occur, right ? :) - return "@jallib preferred" in data + data = file(board).read() + # well, should not consider comment but... should not occur, right ? :) + return "@jallib preferred" in data def is_test_autoable(test): - return not "@jallib skip-auto" in file(test).read() + return not "@jallib skip-auto" in file(test).read() def generate_samples_for_board(path_to_sample,board,outdir=None): - if outdir: - assert os.path.isdir(outdir), "%s must be a directory when auto-generate samples (this is where samples will be stored)" % outdir - testpath = get_full_test_path(path_to_sample) - samplepath = get_full_sample_path(path_to_sample) - fulltestfiles = find_test_files(testpath) - - picname = os.path.basename(board).split("_")[1] # naming convention - # in automated mode, only board files with "@jallib preferred" are kept. - # this is because there can be multiple boards for a given PIC, but only - # ony being used to auto-generate samples - for test in fulltestfiles: - if not is_test_autoable(test): - print >> sys.stderr, "Skip test '%s' because tagged 'skip-auto'" % test - continue - samplename = picname + "_" + os.path.basename(test)[5:] # remove "test_", naming convention - fullsamplepath = outdir and os.path.join(outdir,samplename) or get_full_sample_path(path_to_sample,picname,samplename) - try: - generate_one_sample(board,test,fullsamplepath) - except Exception,e: - print >> sys.stderr,"Invalid board/test combination: %s" % e - import traceback - print >> sys.stderr, traceback.format_exc() - continue + if outdir: + assert os.path.isdir(outdir), "%s must be a directory when auto-generate samples (this is where samples will be stored)" % outdir + testpath = get_full_test_path(path_to_sample) ***The diff for this file has been truncated for email.*** --~--~---------~--~----~------------~-------~--~----~ You received this message because you are subscribed to the Google Groups "jallib" group. To post to this group, send email to [email protected] To unsubscribe from this group, send email to [email protected] For more options, visit this group at http://groups.google.com/group/jallib?hl=en -~----------~----~----~----~------~----~------~--~---
