In perl.git, the branch blead has been updated <http://perl5.git.perl.org/perl.git/commitdiff/d52b857690116b4b653e6dd1781067e8c72e6f62?hp=f624cb736a20c433d4e81c202222fc4ff50afe4c>
- Log ----------------------------------------------------------------- commit d52b857690116b4b653e6dd1781067e8c72e6f62 Author: Hugo van der Sanden <[email protected]> Date: Thu Feb 12 00:11:31 2015 +0000 fix assertions for UTF8_TWO_BYTE_HI/LO Replace the stricter MAX_PORTABLE_UTF8_TWO_BYTE check with a looser MAX_UTF8_TWO_BYTE check, else we can't correctly convert codepoints in the range 0x400-0x7ff from utf16 to utf8 on non-ebcdic platforms. ----------------------------------------------------------------------- Summary of changes: t/comp/utf.t | 16 +++++++++------- t/io/bom.t | 13 ++++++++++++- utf8.h | 6 +++--- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/t/comp/utf.t b/t/comp/utf.t index f5190f9..4e747c4 100644 --- a/t/comp/utf.t +++ b/t/comp/utf.t @@ -1,6 +1,6 @@ #!./perl -w -print "1..4016\n"; +print "1..4216\n"; my $test = 0; my %templates = ( @@ -67,15 +67,17 @@ for my $bom (0, 1) { # handles are not implicitly "use utf8", but don't FIXME that # right now, as here we're testing the input filter itself. - for my $expect ("N", "\xFF", "\x{100}", "\x{010a}", "\x{0a23}", - "\x{10000}", "\x{64321}", "\x{10FFFD}", - "\x{1000a}", # 0xD800 0xDC0A - "\x{12800}", # 0xD80A 0xDC00 - ) { + for my $expect ( + "N", "\x{010a}", "\x{0a23}", "\x{64321}", "\x{10FFFD}", + "\x{1000a}", # 0xD800 0xDC0A + "\x{12800}", # 0xD80A 0xDC00 + # explore a bunch of bit-width boundaries + map { chr((1 << $_) - 1), chr(1 << $_) } 7 .. 20 + ) { # A space so that the UTF-16 heuristic triggers - " '" gives two # characters of ASCII. my $write = " '$expect'"; - my $name = 'chrs ' . join ', ', map {ord $_} split '', $expect; + my $name = 'chrs ' . join ', ', map {sprintf "%#x", ord $_} split '', $expect; test($enc, $write, $expect, $bom, $nl, $name); } diff --git a/t/io/bom.t b/t/io/bom.t index dd1be65..70caf53 100644 --- a/t/io/bom.t +++ b/t/io/bom.t @@ -7,8 +7,19 @@ BEGIN { BEGIN { require "./test.pl"; } -plan(tests => 1); +plan(tests => 3); # It is important that the script contains at least one newline character # that can be expanded to \r\n on DOSish systems. fresh_perl_is("\xEF\xBB\xBFprint 1;\nprint 2", "12", {}, "script starts with a BOM" ); + +# Big- and little-endian UTF-16 +for my $end (0, 1) { + my $encoding = $end ? 'UTF-16LE' : 'UTF-16BE'; + my $prog = join '', map chr($_), map { + $end ? @$_[0, 1] : @$_[1, 0] + } ( + [ 0xFE, 0xFF ], map [ 0, ord($_) ], split //, "print 1;\nprint 2" + ); + fresh_perl_is($prog, "12", {}, "BOM indicates $encoding"); +} diff --git a/utf8.h b/utf8.h index aaf878c..8418055 100644 --- a/utf8.h +++ b/utf8.h @@ -393,15 +393,15 @@ Perl's extended UTF-8 means we can have start bytes up to FF. * code point whose UTF-8 is known to occupy 2 bytes; they are less efficient * than the EIGHT_BIT versions on EBCDIC platforms. We use the logical '~' * operator instead of "<=" to avoid getting compiler warnings. - * MAX_PORTABLE_UTF8_TWO_BYTE should be exactly all one bits in the lower few + * MAX_UTF8_TWO_BYTE should be exactly all one bits in the lower few * places, so the ~ works */ #define UTF8_TWO_BYTE_HI(c) \ (__ASSERT_((sizeof(c) == 1) \ - || !(((WIDEST_UTYPE)(c)) & ~MAX_PORTABLE_UTF8_TWO_BYTE)) \ + || !(((WIDEST_UTYPE)(c)) & ~MAX_UTF8_TWO_BYTE)) \ ((U8) __BASE_TWO_BYTE_HI(c, NATIVE_TO_UNI))) #define UTF8_TWO_BYTE_LO(c) \ (__ASSERT_((sizeof(c) == 1) \ - || !(((WIDEST_UTYPE)(c)) & ~MAX_PORTABLE_UTF8_TWO_BYTE)) \ + || !(((WIDEST_UTYPE)(c)) & ~MAX_UTF8_TWO_BYTE)) \ ((U8) __BASE_TWO_BYTE_LO(c, NATIVE_TO_UNI))) /* This is illegal in any well-formed UTF-8 in both EBCDIC and ASCII -- Perl5 Master Repository
