On 06/04/15 08:42, Jordan Justen wrote: > Supplementary Plane characters can exist in UTF-16 files, > but they are not valid UCS-2 characters. > > For example, refer to this python interpreter code: >>>> import codecs >>>> codecs.encode(u'\U00010300', 'utf-16') > '\xff\xfe\x00\xd8\x00\xdf' > > Therefore the UCS-4 0x00010300 character is encoded as two > 16-bit numbers (0xd800 0xdf00) in a little endian UTF-16 > file. > > For more information, see: > http://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF > > This means that our current BaseTools code could be allowing > unsupported UTF-16 characters be used. To fix this, we decode the file > using python's utf-16 decode support. Then we verify that each > character's code point is 0xffff or less. > > v3: > * Based on Mike Kinney's feedback, we now read the whole file and > verify up-front that it contains valid UCS-2 characters. Thanks > also to Laszlo Ersek for pointing out the Supplementary Plane > characters. > > v4: > * Reject code points in 0xd800-0xdfff range since they are reserved > for UTF-16 surrogate pairs. (lersek) > > Contributed-under: TianoCore Contribution Agreement 1.0 > Signed-off-by: Jordan Justen <jordan.l.jus...@intel.com> > Cc: Yingke D Liu <yingke.d....@intel.com> > Cc: Michael D Kinney <michael.d.kin...@intel.com> > Cc: Laszlo Ersek <ler...@redhat.com> > --- > BaseTools/Source/Python/AutoGen/UniClassObject.py | 88 > ++++++++++++++++++++++- > 1 file changed, 86 insertions(+), 2 deletions(-) > > diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py > b/BaseTools/Source/Python/AutoGen/UniClassObject.py > index aa54f4f..386e1ec 100644 > --- a/BaseTools/Source/Python/AutoGen/UniClassObject.py > +++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py > @@ -19,6 +19,7 @@ > import Common.LongFilePathOs as os, codecs, re > import distutils.util > import Common.EdkLogger as EdkLogger > +import StringIO > from Common.BuildToolError import * > from Common.String import GetLineNo > from Common.Misc import PathClass > @@ -147,6 +148,37 @@ def GetLanguageCode(LangName, IsCompatibleMode, File): > > EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 > language code : %s" % LangName, File) > > +## Ucs2Codec > +# > +# This is only a partial codec implementation. It only supports > +# encoding, and is primarily used to check that all the characters are > +# valid for UCS-2. > +# > +class Ucs2Codec(codecs.Codec): > + def __init__(self): > + self.__utf16 = codecs.lookup('utf-16') > + > + def encode(self, input, errors='strict'): > + for Char in input: > + CodePoint = ord(Char) > + if CodePoint >= 0xd800 and CodePoint <= 0xdfff: > + raise ValueError("Code Point is in range reserved for " + > + "UTF-16 surrogate pairs") > + elif CodePoint > 0xffff: > + raise ValueError("Code Point too large to encode in UCS-2") > + return self.__utf16.encode(input) > + > +TheUcs2Codec = Ucs2Codec() > +def Ucs2Search(name): > + if name == 'ucs-2': > + return codecs.CodecInfo( > + name=name, > + encode=TheUcs2Codec.encode, > + decode=TheUcs2Codec.decode) > + else: > + return None > +codecs.register(Ucs2Search) > + > ## StringDefClassObject > # > # A structure for language definition > @@ -209,7 +241,7 @@ class UniFileClassObject(object): > Lang = distutils.util.split_quoted((Line.split(u"//")[0])) > if len(Lang) != 3: > try: > - FileIn = codecs.open(LongFilePath(File.Path), mode='rb', > encoding='utf-16').read() > + FileIn = self.OpenUniFile(LongFilePath(File.Path)) > except UnicodeError, X: > EdkLogger.error("build", FILE_READ_FAILURE, "File read > failure: %s" % str(X), ExtraData=File); > except: > @@ -253,6 +285,58 @@ class UniFileClassObject(object): > self.OrderedStringDict[LangName][Item.StringName] = > len(self.OrderedStringList[LangName]) - 1 > return True > > + def OpenUniFile(self, FileName): > + # > + # Read file > + # > + try: > + UniFile = open(FileName, mode='rb') > + FileIn = UniFile.read() > + UniFile.close() > + except: > + EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File) > + > + # > + # We currently only support UTF-16 > + # > + Encoding = 'utf-16' > + > + self.VerifyUcs2Data(FileIn, FileName, Encoding) > + > + UniFile = StringIO.StringIO(FileIn) > + Info = codecs.lookup(Encoding) > + (Reader, Writer) = (Info.streamreader, Info.streamwriter) > + return codecs.StreamReaderWriter(UniFile, Reader, Writer) > + > + def VerifyUcs2Data(self, FileIn, FileName, Encoding): > + Ucs2Info = codecs.lookup('ucs-2') > + # > + # Convert to unicode > + # > + try: > + FileDecoded = codecs.decode(FileIn, Encoding) > + Ucs2Info.encode(FileDecoded) > + except: > + UniFile = StringIO.StringIO(FileIn) > + Info = codecs.lookup(Encoding) > + (Reader, Writer) = (Info.streamreader, Info.streamwriter) > + File = codecs.StreamReaderWriter(UniFile, Reader, Writer) > + LineNumber = 0 > + ErrMsg = lambda Encoding, LineNumber: \ > + '%s contains invalid %s characters on line %d.' % \ > + (FileName, Encoding, LineNumber) > + while True: > + LineNumber = LineNumber + 1 > + try: > + Line = File.readline() > + if Line == '': > + EdkLogger.error('Unicode File Parser', PARSER_ERROR, > + ErrMsg(Encoding, LineNumber)) > + Ucs2Info.encode(Line) > + except: > + EdkLogger.error('Unicode File Parser', PARSER_ERROR, > + ErrMsg('UCS-2', LineNumber)) > + > # > # Get String name and value > # > @@ -305,7 +389,7 @@ class UniFileClassObject(object): > EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, > ExtraData=File.Path) > > try: > - FileIn = codecs.open(LongFilePath(File.Path), mode='rb', > encoding='utf-16') > + FileIn = self.OpenUniFile(LongFilePath(File.Path)) > except UnicodeError, X: > EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: > %s" % str(X), ExtraData=File.Path); > except: >
Reviewed-by: Laszlo Ersek <ler...@redhat.com> ------------------------------------------------------------------------------ _______________________________________________ edk2-devel mailing list edk2-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/edk2-devel
