On Fri, Jun 21, 2019 at 01:58:57AM +0000, Fan, ZhijuX wrote: > BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1855 > > UniTool is one python script to generate UQI (Universal Question > Identifier) unicode string for HII question PROMPT string. UQI > string can be used to identify each HII question. > The scripts function will sync up UQI definitions with uni files > based on vfi/vfr/hfr/sd/sdi in the tree. > > This script can be run in both Py2 and Py3. > > Cc: Liming Gao <liming....@intel.com> > Cc: Bob Feng <bob.c.f...@intel.com> > Cc: Ard Biesheuvel <ard.biesheu...@linaro.org> > Cc: Leif Lindholm <leif.lindh...@linaro.org> > Cc: Michael D Kinney <michael.d.kin...@intel.com> > Signed-off-by: Zhiju.Fan <zhijux....@intel.com> > --- > Platform/Intel/Tools/UniTool/UniTool.py | 515 > ++++++++++++++++++++++++++++++++ > 1 file changed, 515 insertions(+) > create mode 100644 Platform/Intel/Tools/UniTool/UniTool.py > > diff --git a/Platform/Intel/Tools/UniTool/UniTool.py > b/Platform/Intel/Tools/UniTool/UniTool.py > new file mode 100644 > index 0000000000..aec25b51c2 > --- /dev/null > +++ b/Platform/Intel/Tools/UniTool/UniTool.py > @@ -0,0 +1,515 @@ > +## @file > +# generate UQI (Universal Question Identifier) unicode string for HII > question PROMPT string. UQI string can be used to > +# identify each HII question. > +# > +# Copyright (c) 2019, Intel Corporation. All rights reserved. > +# > +# SPDX-License-Identifier: BSD-2-Clause-Patent > +# > + > +import re > +import sys > +import os > +import getopt > +import codecs > +import fnmatch > +import logging > + > +# global variable declarations > +QuestionError = False > +UqiList = re.compile('^#string[ \t]+([A-Z_0-9]+)[ \t]+#language[ \t]+uqi[ > \t\r\n]+"(?:[x\S]{1,2})([0-9a-fA-F]{4,5})"', > + re.M).findall > +AllUqis = {} > +StringDict = {} > +GlobalVarId = {} > +Options = {} > + > + > +# ********************************************************************** > +# description: Prints help information > +# > +# arguments: none > +# > +# returns: none > +# > + > +def Usage():
Why not use argparse? / Leif > + print("Syntax: %s [-b] [-u] [-l] [-x] [-h] [-d 'rootDirectory1'] [-d > 'rootDirectory2'] [-d 'rootDirectory3']... [-q e|w] \ > +'rootDirectory0' 'uqiFile'|'uqiFileDirectory' ['excludedDirectory1'] > ['excludedDirectory2'] ['excludedDirectory3']...\n%s" % > + (os.path.basename(sys.argv[0]), > + """\nFunction will sync up UQI definitions with uni files based > on vfi/vfr/hfr/sd/sdi in the tree.\n > + Required Arguments: > + 'rootdirectory0' path to root directory > + 'uqiFileDirectory' path to UQI file(UqiList.uni) > + 'uqiFile' UQI file > + > + Options: > + -h Show this help > + -b Build option returns error if any new UQI needs > assigning > + based on vfi/vfr/hfr/sd/sdi when no -u option > is specified > + -u Create new UQIs that does not already exist in > uqiFile for > + any string requiring a UQI based on > vfi/vfr/hfr/sd/sdi > + NOTE: 'uqiFile' cannot be readonly! > + -l Language deletion option (keeps only English > and uqi) > + moves all UQIs to 'uqiFile' > + NOTE: Uni files cannot be readonly! > + -x Exclude 'rootDirectory'/'excludedDirectory1' & > + 'rootDirectory'/'excludedDirectory2'... from > UQI list build > + NOTE: Cannot be the same as rootDirectory > + -d Add multiple root directories to process > + -q Print warning(w) or return error(e) if > different HII questions > + are referring same string token > + > + Return error if any duplicated UQI string or value in UQI list or if no > definition > + for any string referred by HII question when -b or -u is specified > + > + NOTE: Options must be specified before parameters > + """)) > + sys.exit() > + > + > +# ********************************************************************** > +# description: Get uni file encoding > +# > +# arguments: Filename - name of uni file > +# > +# returns: utf-8 or utf-16 > +# > +def GetUniFileEncoding(Filename): > + # > + # Detect Byte Order Mark at beginning of file. Default to UTF-8 > + # > + Encoding = 'utf-8' > + > + # > + # Read file > + # > + try: > + with open(Filename, mode='rb') as UniFile: > + FileIn = UniFile.read() > + except: > + return Encoding > + > + if (FileIn.startswith(codecs.BOM_UTF16_BE) or > FileIn.startswith(codecs.BOM_UTF16_LE)): > + Encoding = 'utf-16' > + > + return Encoding > + > + > +# rewrite function os.path.walk > +def Walk(Top, Func, Arg): > + try: > + Names = os.listdir(Top) > + except os.error: > + return > + Func(Arg, Top, Names) > + for Name in Names: > + Name = os.path.join(Top, Name) > + if os.path.isdir(Name): > + Walk(Name, Func, Arg) > + > + > +# ********************************************************************** > +# description: Parses commandline arguments and options > +# Calls function processUni to build dictionary of strings > +# Calls other functions according to user specified options > +# > +# arguments: argv - contains all input from command line > +# - must contain path to root directory > +# - may contain options -h, -u, -l, -b or -x before path > +# > +# returns: none > +# > +def main(argv): > + ##### Read input arguments and options > + global AllUqis, UqiList, QuestionError > + try: > + Opts, Args = getopt.getopt(argv[1:], "hulbxd:q:") # each letter is > an optional argument > + except getopt.GetoptError: > + Usage() > + try: > + DirNameList = [Args[0]] > + QuestionOption = None > + for EachOpt in Opts: > + if EachOpt[0] == '-d': > + DirNameList.append(EachOpt[1]) > + if EachOpt[0] == '-q': > + QuestionOption = EachOpt[1] > + if (QuestionOption != "e") and (QuestionOption != "w"): > + print("\nERROR: invalid option value for -q option\n") > + return > + Destname = Args[1] > + if len(Args) > 2: > + ExDirList = Args[2:] > + except: > + Usage() > + > + UpdateUQIs = False > + LangOption = False > + BuildOption = False > + ExcludeOption = False > + ExPathList = [] > + > + for Opt, Args in Opts: > + if Opt == "-h": > + Usage() > + if Opt == "-b": > + BuildOption = True > + if Opt == "-u": > + BuildOption = True > + UpdateUQIs = True > + if Opt == "-l": > + LangOption = True > + if Opt == "-x": > + ExcludeOption = True > + try: > + for EachExDir in ExDirList: > + for EachRootDir in DirNameList: > + if EachExDir == EachRootDir: > + print("\nERROR: excludedDirectory is same as > rootDirectory\n") > + return > + ExPathList.append(EachRootDir + os.sep + EachExDir) > + except: > + Usage() > + > + global Options > + Options = {'Destname': Destname, 'DirNameList': DirNameList, > 'ExPathList': ExPathList, 'BuildOption': BuildOption, > + 'UpdateUQIs': UpdateUQIs, 'LangOption': LangOption, > 'ExcludeOption': ExcludeOption, > + 'QuestionOption': QuestionOption} > + print("UQI file: %s" % Destname) > + for EachDirName in DirNameList: > + Walk(EachDirName, processUni, None) > + if QuestionError: > + return > + if os.path.isdir(Options['Destname']): > + DestFileName = Options['Destname'] + os.sep + 'UqiList.uni' > + else: > + DestFileName = Options['Destname'] > + if os.path.exists(DestFileName) and (DestFileName not in > list(AllUqis.keys())): > + try: > + Encoding = GetUniFileEncoding(DestFileName) > + with codecs.open(DestFileName, 'r+', Encoding) as destFile: > + DestFileBuffer = destFile.read() > + except IOError as e: > + print("ERROR: " + e.args[1]) > + return > + AllUqis[DestFileName] = UqiList(DestFileBuffer) > + if BuildOption: > + ReturnVal = newUqi() > + if (ReturnVal == 1): > + print('Please fix UQI ERROR(s) above before proceeding.') > + else: > + print("No UQI issues detected\n") > + return > + > + > +# ********************************************************************** > +# description: newUqi collects a list of all currently used uqi values in > the tree > +# Halt build if any duplicated string or value in UQI list. > +# If -u option was specified, creates new UQIs that does not > +# already exist in uqiFile for any string requiring a UQI. > +# > +# arguments: none > +# > +# returns: 0 on success > +# 1 on error - this should cause the build to halt > +# > + > +Syntax = "S" > +SyntaxRE = re.compile('#string[ \t]+[A-Z_0-9]+[ \t]+#language[ \t]+uqi[ > \t\r\n]+"([x\S]{1,2}).*', re.DOTALL).findall > + > + > +def newUqi(): > + global Options, GlobalVarId, AllUqis, Syntax, SyntaxRE > + UqiRange = [] > + UqiStringList = [] > + CreateUQI = [] > + ReturnVal = 0 > + BaseNumSpaces = 47 # Used to line up the UQI values in the resulting > uqiFile > + > + # Look for duplication in the current UQIs and collect current range of > UQIs > + for path in AllUqis.keys(): > + for UqiString in AllUqis[path]: # path contains the path and > Filename of each uni file > + # Checks for duplicated strings in UQI list > + for TempString in UqiStringList: > + if TempString == UqiString[0]: > + print("ERROR: UQI string %s was assigned more than once > and will cause corruption!" % UqiString[0]) > + print("Delete one occurrence of the string and rerun > tool.") > + ReturnVal = 1 # halt build > + > + UqiStringList.append(UqiString[0]) > + > + # Checks for duplicated UQI values in UQI list > + if int(UqiString[1], 16) in UqiRange: > + print("ERROR: UQI value %04x was assigned more than once and > will cause corruption!" % int(UqiString[1], > + > 16)) > + print("Delete one occurrance of the UQI and rerun tool to > create alternate value.") > + ReturnVal = 1 # halt build > + UqiRange.append(int(UqiString[1], 16)) > + > + for StringValue in GlobalVarId.keys(): > + StringFound = False > + for path in StringDict.keys(): > + for UniString in StringDict[path]: # path contains the path and > Filename of each uni file > + if (StringValue == UniString): > + StringFound = True > + break > + if not StringFound: > + print("ERROR: No definition for %s referred by HII question" % > (StringValue)) > + ReturnVal = 1 # halt build > + > + # Require a UQI for any string in vfr/vfi files > + for StringValue in GlobalVarId.keys(): > + # Ignore strings defined as STRING_TOKEN(0) > + if (StringValue != "0"): > + # Check if this string already exists in the UQI list > + if (StringValue not in UqiStringList) and (StringValue not in > CreateUQI): > + CreateUQI.append(StringValue) > + if not Options['UpdateUQIs']: > + print("ERROR: No UQI for %s referred by HII question" % > (StringValue)) > + ReturnVal = 1 # halt build after printing all error > messages > + > + if (ReturnVal == 1): > + return ReturnVal > + > + # Update uqiFile with necessary UQIs > + if Options['UpdateUQIs'] and CreateUQI: > + if os.path.isdir(Options['Destname']): > + DestFileName = Options['Destname'] + os.sep + 'UqiList.uni' > + else: > + DestFileName = Options['Destname'] > + try: > + Encoding = GetUniFileEncoding(DestFileName) > + with codecs.open(DestFileName, 'r+', Encoding) as OutputFile: > + PlatformUQI = OutputFile.read() > + except IOError as e: > + print("ERROR: " + e.args[1]) > + if (e.args[0] == 2): > + try: > + with codecs.open(DestFileName, 'w', Encoding) as > OutputFile: > + print(DestFileName + " did not exist. Creating new > file.") > + PlatformUQI = '' > + except: > + print("Error creating " + DestFileName + ".") > + return 1 > + if (e.args[1] == "Permission denied"): > + print( > + "\n%s is Readonly. You must uncheck the ReadOnly > attibute to run the -u option.\n" % DestFileName) > + return 1 > + > + # Determines and sets the UQI number format > + # TODO: there is probably a more elegant way to do this... > + SyntaxL = SyntaxRE(PlatformUQI) > + if len(SyntaxL) != 0: > + Syntax = SyntaxL[0] > + > + # script is reading the file in and writing it back instead of > appending because the codecs module > + # automatically adds a BOM wherever you start writing. This caused > build failure. > + UqiRange.sort() > + if (UqiRange == []): > + NextUqi = 0 > + else: > + NextUqi = UqiRange[len(UqiRange) - 1] + 1 > + > + for StringValue in CreateUQI: > + print("%s will be assigned a new UQI value" % StringValue) > + UqiRange.append(NextUqi) > + # > + # Lines up the UQI values in the resulting uqiFile > + # > + Spaces = " " * (BaseNumSpaces - len(StringValue)) > + PlatformUQI += '#string %s%s #language uqi \"%s%04x\"\r\n' % > (StringValue, Spaces, Syntax, NextUqi) > + print("#string %s%s #language uqi \"%s%04X\"" % (StringValue, > Spaces, Syntax, NextUqi)) > + NextUqi += 1 > + > + with codecs.open(DestFileName, 'r+', Encoding) as OutputFile: > + OutputFile.seek(0) > + OutputFile.write(PlatformUQI) > + > + return 0 > + > + > +# ********************************************************************** > +# description: Parses each uni file to collect dictionary of strings > +# Removes additional languages and overwrites current uni files > +# if -l option was specified > +# > +# arguments: path - directory location of file including file name > +# Filename - name of file to be modified > +# > +# returns: error string if failure occurred; > +# none if completed sucessfully > +# > +# the following are global so that parsefile is quicker > + > +FindUniString = re.compile( > + '^#string[ \t]+([A-Z_0-9]+)(?:[ \t\r\n]+#language[ \t]+[a-zA-Z-]{2,5}[ > \t\r\n]+".*"[ \t]*[\r]?[\n]?)*', > + re.M).findall > + > +OtherLang = re.compile( > + '^#string[ \t]+[A-Z_0-9]+(?:[ \t\r\n]+#language[ \t]+[a-zA-Z-]{2,5}[ > \t\r\n]+".*"[ \t]*[\r]?[\n]?)*', re.M).findall > +EachLang = re.compile('[ \t\r\n]+#language[ \t]+([a-zA-Z-]{2,5})[ > \t\r\n]+".*"[ \t]*[\r]?[\n]?').findall > + > +UqiStrings = re.compile('^#string[ \t]+[A-Z_0-9]+[ \t]+#language[ \t]+uqi[ > \t\r\n]+".*"[ \t]*[\r]?[\n]?', re.M) > + > + > +def parsefile(path, Filename): > + global Options, StringDict, AllUqis, UqiList, FindUniString, OtherLang, > EachLang, UqiStrings > + > + FullPath = path + os.sep + Filename > + > + try: > + UniEncoding = GetUniFileEncoding(FullPath) > + with codecs.open(FullPath, 'r', UniEncoding) as UniFile: > + Databuffer = UniFile.read() > + except: > + print("Error opening " + FullPath + " for reading.") > + return > + WriteFile = False > + > + if os.path.isdir(Options['Destname']): > + DestFileName = Options['Destname'] + os.sep + 'UqiList.uni' > + else: > + DestFileName = Options['Destname'] > + > + if Options['LangOption']: > + try: > + UqiEncoding = GetUniFileEncoding(DestFileName) > + with codecs.open(DestFileName, 'r+', UqiEncoding) as OutputFile: > + PlatformUQI = OutputFile.read() > + except IOError as e: > + print("ERROR: " + e.args[1]) > + if (e.args[0] == 2): > + try: > + with codecs.open(DestFileName, 'w', UqiEncoding) as > OutputFile: > + print(DestFileName + " did not exist. Creating new > file.") > + PlatformUQI = '' > + except: > + print("Error creating " + DestFileName + ".") > + return > + else: > + print("Error opening " + DestFileName + " for appending.") > + return > + > + if (Filename != DestFileName.split(os.sep)[-1]): > + Uqis = re.findall(UqiStrings, Databuffer) > + if Uqis: > + for Uqi in Uqis: > + PlatformUQI += Uqi > + with codecs.open(DestFileName, 'r+', UqiEncoding) as > OutputFile: > + OutputFile.seek(0) > + OutputFile.write(PlatformUQI) > + Databuffer = re.sub(UqiStrings, '', Databuffer) > + if Uqis: > + WriteFile = True > + print("Deleted uqis from %s" % FullPath) > + stringlist = OtherLang(Databuffer) > + for stringfound in stringlist: > + ThisString = EachLang(stringfound) > + for LanguageFound in ThisString: > + if ((LanguageFound != 'en') and (LanguageFound != > 'en-US') and (LanguageFound != 'eng') and ( > + LanguageFound != 'uqi')): > + Databuffer = re.sub(re.escape(stringfound), '', > Databuffer) > + WriteFile = True > + print("Deleted %s from %s" % (LanguageFound, > FullPath)) > + if (Filename != DestFileName.split(os.sep)[-1]): > + # adding strings to dictionary > + StringDict[r'%s' % FullPath] = FindUniString(Databuffer) > + # adding UQIs to dictionary > + AllUqis[r'%s' % FullPath] = UqiList(Databuffer) > + > + if WriteFile: > + try: > + with codecs.open(FullPath, 'w', UniEncoding) as UniFile: > + UniFile.write(Databuffer) > + except: > + print("Error opening " + FullPath + " for writing.") > + return > + > + > +# ********************************************************************** > +# description: Searches tree for uni files > +# Calls parsefile to collect dictionary of strings in each uni > file > +# Calls searchVfiFile for each vfi or vfr file found > +# > +# arguments: argument list is built by os.path.walk function call > +# arg - None > +# dirname - directory location of files > +# names - specific files to search in directory > +# > +# returns: none > +# > +def processUni(args, dirname, names): > + global Options > + # Remove excludedDirectory > + if Options['ExcludeOption']: > + for EachExDir in Options['ExPathList']: > + for dir in names: > + if os.path.join(dirname, dir) == EachExDir: > + names.remove(dir) > + > + for entry in names: > + FullPath = dirname + os.sep + entry > + if fnmatch.fnmatch(FullPath, '*.uni'): > + parsefile(dirname, entry) > + if fnmatch.fnmatch(FullPath, '*.vf*'): > + searchVfiFile(FullPath) > + if fnmatch.fnmatch(FullPath, '*.sd'): > + searchVfiFile(FullPath) > + if fnmatch.fnmatch(FullPath, '*.sdi'): > + searchVfiFile(FullPath) > + if fnmatch.fnmatch(FullPath, '*.hfr'): > + searchVfiFile(FullPath) > + return > + > + > +# ********************************************************************** > +# description: Compose a dictionary of all strings that may need UQIs > assigned > +# to them and key is the string > +# > +# arguments: Filename - name of file to search for strings > +# > +# returns: none > +# > + > +# separate regexes for readability > +StringGroups = re.compile( > + '^[ \t]*(?:oneof|numeric|checkbox|orderedlist)[ > \t]+varid.+?(?:endoneof|endnumeric|endcheckbox|endorderedlist);', > + re.DOTALL | re.M).findall > +StringVarIds = re.compile( > + '[ \t]*(?:oneof|numeric|checkbox|orderedlist)[ \t]+varid[ \t]*=[ > \t]*([a-zA-Z_0-9]+\.[a-zA-Z_0-9]+)').findall > +StringTokens = re.compile('prompt[ \t]*=[ \t]*STRING_TOKEN[ > \t]*\(([a-zA-Z_0-9]+)\)').findall > + > + > +def searchVfiFile(Filename): > + global Options, GlobalVarId, StringGroups, StringVarIds, StringTokens, > QuestionError > + try: > + with open(Filename, 'r') as VfiFile: > + Databuffer = VfiFile.read() > + > + # Finds specified lines in file > + VfiStringGroup = StringGroups(Databuffer) > + > + # Searches for prompts within specified lines > + for EachGroup in VfiStringGroup: > + for EachString in StringTokens(EachGroup): > + # Ignore strings defined as STRING_TOKEN(0), > STRING_TOKEN(STR_EMPTY) or STRING_TOKEN(STR_NULL) > + if (EachString != "0") and (EachString != "STR_EMPTY") and > (EachString != "STR_NULL"): > + if EachString not in GlobalVarId: > + GlobalVarId[EachString] = StringVarIds(EachGroup) > + else: > + if (GlobalVarId[EachString][0] != > StringVarIds(EachGroup)[0]): > + if Options['QuestionOption']: > + if Options['QuestionOption'] == "e": > + QuestionError = True > + print("ERROR:"), > + if Options['QuestionOption'] == "w": > + print("WARNING:"), > + print("%s referred by different HII > questions(%s and %s)" % ( > + EachString, GlobalVarId[EachString][0], > StringVarIds(EachGroup)[0])) > + except: > + print("Error opening file at %s for reading." % Filename) > + > + > +if __name__ == '__main__': > + sys.exit(main(sys.argv)) > -- > 2.14.1.windows.1 > -=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#42691): https://edk2.groups.io/g/devel/message/42691 Mute This Topic: https://groups.io/mt/32154576/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-