Reviewed-by: Yonghong Zhu <[email protected]> Best Regards, Zhu Yonghong
-----Original Message----- From: Feng, YunhuaX Sent: Tuesday, February 27, 2018 4:43 PM To: [email protected] Cc: Zhu, Yonghong <[email protected]>; Gao, Liming <[email protected]> Subject: [PATCH V2]BaseTools: Fix flexible PCD single quote and double quote bugs V2: 1. Fix parse DSC file split PCD Value {'9','a','#'} build error 2. Fix parse --pcd gUefiOvmfPkgTokenSpaceGuid.test10="\'A\'" build error, If PCD type is UINT8 or UINT16 or UINT32 or UINT64 V1: 1. Argument --pcd format as below: Some examples that to match --pcd format and DSC format --pcd DSC format L"ABC" L"ABC" "AB\\\"C" "AB\"C" "AB\tC" "AB\tC" "AB\\\'C" "AB\'C" "\'ABC\'" 'ABC’ L"\'AB\\\"C\'" L'AB\"C' "\'AB\\\'C\'" 'AB\'C' "\'AB\tC\'" 'AB\tC' H"{0, L\"AB\\\"B\", \'ab\\\"c\'}" {0, L"AB\"B", 'ab\"c'} H"{0, L\"AB\\\"B\",\'ab\\\'c\'}" {0, L"AB\"B", 'ab\'c'} 2. {'#', '|'} split string incorrectly. Cc: Liming Gao <[email protected]> Cc: Yonghong Zhu <[email protected]> Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Yunhua Feng <[email protected]> --- BaseTools/Source/Python/AutoGen/GenMake.py | 20 +++++--- BaseTools/Source/Python/Common/Expression.py | 60 ++++++++++++++++------- BaseTools/Source/Python/Common/Misc.py | 31 +++++++----- BaseTools/Source/Python/Common/String.py | 46 ++++++++++------- BaseTools/Source/Python/Workspace/DscBuildData.py | 31 +++++++----- 5 files changed, 123 insertions(+), 65 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index afe6f2f99c..4b924d21e0 100644 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -1553,17 +1553,23 @@ class TopLevelMakefile(BuildFile): if GlobalData.BuildOptionPcd: for index, option in enumerate(GlobalData.gCommand): if "--pcd" == option and GlobalData.gCommand[index+1]: pcdName, pcdValue = GlobalData.gCommand[index+1].split('=') - if pcdValue.startswith('H'): - pcdValue = 'H' + '"' + pcdValue[1:] + '"' - ExtraOption += " --pcd " + pcdName + '=' + pcdValue - elif pcdValue.startswith("L'"): - ExtraOption += "--pcd " + pcdName + '=' + pcdValue - elif pcdValue.startswith('L'): - pcdValue = 'L' + '"' + pcdValue[1:] + '"' + for Item in GlobalData.BuildOptionPcd: + if '.'.join(Item[0:2]) == pcdName: + pcdValue = Item[2] + if pcdValue.startswith('L') or pcdValue.startswith('"'): + pcdValue, Size = ParseFieldValue(pcdValue) + NewVal = '{' + for S in range(Size): + NewVal = NewVal + '0x%02X' % ((pcdValue >> S * 8) & 0xff) + NewVal += ',' + pcdValue = NewVal[:-1] + '}' + break + if pcdValue.startswith('{'): + pcdValue = 'H' + '"' + pcdValue + '"' ExtraOption += " --pcd " + pcdName + '=' + pcdValue else: ExtraOption += " --pcd " + GlobalData.gCommand[index+1] MakefileName = self._FILE_NAME_[self._FileType] diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py index edb0a60de6..f1516d5c7b 100644 --- a/BaseTools/Source/Python/Common/Expression.py +++ b/BaseTools/Source/Python/Common/Expression.py @@ -43,28 +43,41 @@ ERR_IN_OPERAND = 'Macro after IN operator can only be: $(FAMILY), $(ARC ## SplitString # Split string to list according double quote # For example: abc"de\"f"ghi"jkl"mn will be: ['abc', '"de\"f"', 'ghi', '"jkl"', 'mn'] # def SplitString(String): - # There might be escaped quote: "abc\"def\\\"ghi" - Str = String.replace('\\\\', '//').replace('\\\"', '\\\'') + # There might be escaped quote: "abc\"def\\\"ghi", 'abc\'def\\\'ghi' + Str = String RetList = [] - InQuote = False + InSingleQuote = False + InDoubleQuote = False Item = '' for i, ch in enumerate(Str): - if ch == '"': - InQuote = not InQuote - if not InQuote: + if ch == '"' and not InSingleQuote: + if Str[i - 1] != '\\': + InDoubleQuote = not InDoubleQuote + if not InDoubleQuote: + Item += String[i] + RetList.append(Item) + Item = '' + continue + if Item: + RetList.append(Item) + Item = '' + elif ch == "'" and not InDoubleQuote: + if Str[i - 1] != '\\': + InSingleQuote = not InSingleQuote + if not InSingleQuote: Item += String[i] RetList.append(Item) Item = '' continue if Item: RetList.append(Item) Item = '' Item += String[i] - if InQuote: + if InSingleQuote or InDoubleQuote: raise BadExpression(ERR_STRING_TOKEN % Item) if Item: RetList.append(Item) return RetList @@ -481,17 +494,21 @@ class ValueExpression(object): Radix = 16 if self._Token.startswith('"') or self._Token.startswith('L"'): Flag = 0 for Index in range(len(self._Token)): if self._Token[Index] in ['"']: + if self._Token[Index - 1] == '\\': + continue Flag += 1 if Flag == 2 and self._Token.endswith('"'): return True if self._Token.startswith("'") or self._Token.startswith("L'"): Flag = 0 for Index in range(len(self._Token)): if self._Token[Index] in ["'"]: + if self._Token[Index - 1] == '\\': + continue Flag += 1 if Flag == 2 and self._Token.endswith("'"): return True try: self._Token = int(self._Token, Radix) @@ -535,20 +552,29 @@ class ValueExpression(object): # Skip left quote self._Idx += 1 # Replace escape \\\", \" - Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'') - for Ch in Expr: - self._Idx += 1 - if Ch == '"' or Ch == "'": - break - self._Token = self._LiteralToken = self._Expr[Idx:self._Idx] - if self._Token.startswith('"') and not self._Token.endswith('"'): - raise BadExpression(ERR_STRING_TOKEN % self._Token) - if self._Token.startswith("'") and not self._Token.endswith("'"): - raise BadExpression(ERR_STRING_TOKEN % self._Token) + if self._Expr[Idx] == '"': + Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'') + for Ch in Expr: + self._Idx += 1 + if Ch == '"': + break + self._Token = self._LiteralToken = self._Expr[Idx:self._Idx] + if not self._Token.endswith('"'): + raise BadExpression(ERR_STRING_TOKEN % self._Token) + #Replace escape \\\', \' + elif self._Expr[Idx] == "'": + Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace("\\\'", "\\\"") + for Ch in Expr: + self._Idx += 1 + if Ch == "'": + break + self._Token = self._LiteralToken = self._Expr[Idx:self._Idx] + if not self._Token.endswith("'"): + raise BadExpression(ERR_STRING_TOKEN % self._Token) self._Token = self._Token[1:-1] return self._Token # Get token that is comprised by alphanumeric, underscore or dot(used by PCD) # @param IsAlphaOp: Indicate if parsing general token or script operator(EQ, NE...) diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py index 1461d00669..a7e7797d04 100644 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -1441,25 +1441,30 @@ def ParseConsoleLog(Filename): Opr.close() Opw.close() def AnalyzePcdExpression(Setting): Setting = Setting.strip() - # There might be escaped quote in a string: \", \\\" - Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'') + # There might be escaped quote in a string: \", \\\" , \', \\\' + Data = Setting # There might be '|' in string and in ( ... | ... ), replace it with '-' NewStr = '' - InStr = False + InSingleQuoteStr = False + InDoubleQuoteStr = False Pair = 0 - for ch in Data: - if ch == '"': - InStr = not InStr - elif ch == '(' and not InStr: + for Index, ch in enumerate(Data): + if ch == '"' and not InSingleQuoteStr: + if Data[Index - 1] != '\\': + InDoubleQuoteStr = not InDoubleQuoteStr + elif ch == "'" and not InDoubleQuoteStr: + if Data[Index - 1] != '\\': + InSingleQuoteStr = not InSingleQuoteStr + elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr): Pair += 1 - elif ch == ')' and not InStr: + elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr): Pair -= 1 - if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT: + if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT: NewStr += '-' else: NewStr += ch FieldList = [] StartPos = 0 @@ -1547,37 +1552,37 @@ def ParseFieldValue (Value): raise BadExpression('%s' % Message) Value, Size = ParseFieldValue(Value) return Value, 16 if Value.startswith('L"') and Value.endswith('"'): # Unicode String - List = list(Value[2:-1]) + List = list(eval(Value[1:])) # translate escape character List.reverse() Value = 0 for Char in List: Value = (Value << 16) | ord(Char) return Value, (len(List) + 1) * 2 if Value.startswith('"') and Value.endswith('"'): # ASCII String - List = list(Value[1:-1]) + List = list(eval(Value)) # translate escape character List.reverse() Value = 0 for Char in List: Value = (Value << 8) | ord(Char) return Value, len(List) + 1 if Value.startswith("L'") and Value.endswith("'"): # Unicode Character Constant - List = list(Value[2:-1]) + List = list(eval(Value[1:])) # translate escape character if len(List) == 0: raise BadExpression('Length %s is %s' % (Value, len(List))) List.reverse() Value = 0 for Char in List: Value = (Value << 16) | ord(Char) return Value, len(List) * 2 if Value.startswith("'") and Value.endswith("'"): # Character constant - List = list(Value[1:-1]) + List = list(eval(Value)) # translate escape character if len(List) == 0: raise BadExpression('Length %s is %s' % (Value, len(List))) List.reverse() Value = 0 for Char in List: diff --git a/BaseTools/Source/Python/Common/String.py b/BaseTools/Source/Python/Common/String.py index 4a8c03e88e..5e50beff5c 100644 --- a/BaseTools/Source/Python/Common/String.py +++ b/BaseTools/Source/Python/Common/String.py @@ -43,30 +43,36 @@ gHumanReadableVerPatt = re.compile(r'([1-9][0-9]*|0)\.[0-9]{1,2}$') # def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): ValueList = [] Last = 0 Escaped = False - InString = False + InSingleQuoteString = False + InDoubleQuoteString = False InParenthesis = 0 for Index in range(0, len(String)): Char = String[Index] if not Escaped: # Found a splitter not in a string, split it - if not InString and InParenthesis == 0 and Char == SplitTag: + if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag: ValueList.append(String[Last:Index].strip()) Last = Index + 1 if MaxSplit > 0 and len(ValueList) >= MaxSplit: break - if Char == '\\' and InString: + if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString): Escaped = True - elif Char == '"': - if not InString: - InString = True + elif Char == '"' and not InSingleQuoteString: + if not InDoubleQuoteString: + InDoubleQuoteString = True else: - InString = False + InDoubleQuoteString = False + elif Char == "'" and not InDoubleQuoteString: + if not InSingleQuoteString: + InSingleQuoteString = True + else: + InSingleQuoteString = False elif Char == '(': InParenthesis = InParenthesis + 1 elif Char == ')': InParenthesis = InParenthesis - 1 else: @@ -343,18 +349,21 @@ def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyle if AllowCppStyleComment: Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter) # # remove comments, but we should escape comment character in string # - InString = False + InDoubleQuoteString = False + InSingleQuoteString = False CommentInString = False for Index in range(0, len(Line)): - if Line[Index] == '"': - InString = not InString - elif Line[Index] == CommentCharacter and InString : + if Line[Index] == '"' and not InSingleQuoteString: + InDoubleQuoteString = not InDoubleQuoteString + elif Line[Index] == "'" and not InDoubleQuoteString: + InSingleQuoteString = not InSingleQuoteString + elif Line[Index] == CommentCharacter and (InSingleQuoteString or InDoubleQuoteString): CommentInString = True - elif Line[Index] == CommentCharacter and not InString : + elif Line[Index] == CommentCharacter and not (InSingleQuoteString or InDoubleQuoteString): Line = Line[0: Index] break if CommentInString and BuildOption: Line = Line.replace('"', '') @@ -400,19 +409,22 @@ def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyl if AllowCppStyleComment: Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter) # # separate comments and statements, but we should escape comment character in string # - InString = False + InDoubleQuoteString = False + InSingleQuoteString = False CommentInString = False Comment = '' for Index in range(0, len(Line)): - if Line[Index] == '"': - InString = not InString - elif Line[Index] == CommentCharacter and InString: + if Line[Index] == '"' and not InSingleQuoteString: + InDoubleQuoteString = not InDoubleQuoteString + elif Line[Index] == "'" and not InDoubleQuoteString: + InSingleQuoteString = not InSingleQuoteString + elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString): CommentInString = True - elif Line[Index] == CommentCharacter and not InString: + elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString): Comment = Line[Index:].strip() Line = Line[0:Index].strip() break return Line, Comment diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index 75b877a5aa..e3da4585b3 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -989,10 +989,12 @@ class DscBuildData(PlatformBuildClassObject): FoundFlag = True if FieldName: NewValue = self.GetFieldValueFromComm(pcdvalue, TokenSpaceGuidCName, TokenCName, FieldName) GlobalData.BuildOptionPcd[i] = (TokenSpaceGuidCName, TokenCName, FieldName,NewValue,("build command options",1)) else: + # Replace \' to ', \\\' to \' + pcdvalue = pcdvalue.replace("\\\\\\'", + '\\\\\\"').replace('\\\'', '\'').replace('\\\\\\"', "\\'") for key in self.DecPcds: PcdItem = self.DecPcds[key] if HasTokenSpace: if (PcdItem.TokenCName, PcdItem.TokenSpaceGuidCName) == (TokenCName, TokenSpaceGuidCName): PcdDatumType = PcdItem.DatumType @@ -1000,27 +1002,27 @@ class DscBuildData(PlatformBuildClassObject): try: pcdvalue = ValueExpressionEx(pcdvalue[1:], PcdDatumType, self._GuidDict)(True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) - if PcdDatumType == "VOID*": + if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']: pcdvalue = 'H' + pcdvalue elif pcdvalue.startswith("L'"): try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) - if pcdvalue.startswith('{'): + if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']: pcdvalue = 'H' + pcdvalue elif pcdvalue.startswith("'"): try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) - if pcdvalue.startswith('{'): + if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']: pcdvalue = 'H' + pcdvalue elif pcdvalue.startswith('L'): pcdvalue = 'L"' + pcdvalue[1:] + '"' try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) @@ -1029,12 +1031,16 @@ class DscBuildData(PlatformBuildClassObject): (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) else: try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) except BadExpression, Value: - EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % - (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) + try: + pcdvalue = '"' + pcdvalue + '"' + pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) + except BadExpression, Value: + EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % + + (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue) FoundFlag = True else: if PcdItem.TokenCName == TokenCName: if not PcdItem.TokenSpaceGuidCName in TokenSpaceGuidCNameList: @@ -1046,29 +1052,29 @@ class DscBuildData(PlatformBuildClassObject): try: pcdvalue = ValueExpressionEx(pcdvalue[1:], PcdDatumType, self._GuidDict)(True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) - if PcdDatumType == "VOID*": + if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64,'BOOLEAN']: pcdvalue = 'H' + pcdvalue elif pcdvalue.startswith("L'"): try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)( True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) - if pcdvalue.startswith('{'): + if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']: pcdvalue = 'H' + pcdvalue elif pcdvalue.startswith("'"): try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)( True) except BadExpression, Value: EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) - if pcdvalue.startswith('{'): + if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']: pcdvalue = 'H' + pcdvalue elif pcdvalue.startswith('L'): pcdvalue = 'L"' + pcdvalue[1:] + '"' try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)( @@ -1078,13 +1084,16 @@ class DscBuildData(PlatformBuildClassObject): (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) else: try: pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) except BadExpression, Value: - EdkLogger.error('Parser', FORMAT_INVALID, - 'PCD [%s.%s] Value "%s", %s' % - (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) + try: + pcdvalue = '"' + pcdvalue + '"' + pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True) + except BadExpression, Value: + EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' % + + (TokenSpaceGuidCName, TokenCName, pcdvalue, Value)) NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue) FoundFlag = True else: EdkLogger.error( 'build', -- 2.12.2.windows.2 _______________________________________________ edk2-devel mailing list [email protected] https://lists.01.org/mailman/listinfo/edk2-devel
