In perl.git, the branch blead has been updated <http://perl5.git.perl.org/perl.git/commitdiff/69bc4c1f86bca21cf0baeb2b4812ea97d3bf438e?hp=8d19ebbca9eecf219cc453cffe88722722860dd9>
- Log ----------------------------------------------------------------- commit 69bc4c1f86bca21cf0baeb2b4812ea97d3bf438e Author: Karl Williamson <[email protected]> Date: Sat Aug 27 21:17:49 2016 -0600 Add C macros for UTF-8 for BOM and REPLACEMENT CHARACTER This makes it easy for module authors to write XS code that can use these characters, and be automatically portable to EBCDIC systems. ----------------------------------------------------------------------- Summary of changes: regen/unicode_constants.pl | 31 +++++++++++++++++++++++++++++++ unicode_constants.h | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/regen/unicode_constants.pl b/regen/unicode_constants.pl index baf25f1..acdbaa3 100644 --- a/regen/unicode_constants.pl +++ b/regen/unicode_constants.pl @@ -28,6 +28,30 @@ print $out_fh <<END; * "_TAIL" if instead it represents all but the first byte. This, and * with no additional suffix are both string constants */ +/* +=head1 Unicode Support + +=for apidoc AmU|placeholder|BOM_UTF8 + +This is a macro that evaluates to a string constant of the UTF-8 bytes that +define the Unicode BYTE ORDER MARK (U+FEFF) for the platform that perl +is compiled on. This allows code to use a mnemonic for this character that +works on both ASCII and EBCDIC platforms. +S<C<sizeof(BOM_UTF8) - 1>> can be used to get its length in +bytes. + +=for apidoc AmU|placeholder|REPLACEMENT_CHARACTER_UTF8 + +This is a macro that evaluates to a string constant of the UTF-8 bytes that +define the Unicode REPLACEMENT CHARACTER (U+FFFD) for the platform that perl +is compiled on. This allows code to use a mnemonic for this character that +works on both ASCII and EBCDIC platforms. +S<C<sizeof(REPLACEMENT_CHARACTER_UTF8) - 1>> can be used to get its length in +bytes. + +=cut +*/ + END my $version = Unicode::UCD::UnicodeVersion(); @@ -180,6 +204,9 @@ read_only_bottom_close_and_rename($out_fh); # DATA FORMAT # +# Note that any apidoc comments you want in the file need to be added to one +# of the prints above +# # A blank line is output as-is. # Comments (lines whose first non-blank is a '#') are converted to C-style, # though empty comments are converted to blank lines. Otherwise, each line @@ -228,6 +255,10 @@ U+2010 string BOM first BOM tail +BOM string + +U+FFFD string + NBSP native NBSP string diff --git a/unicode_constants.h b/unicode_constants.h index 02d3811..1c0e62e 100644 --- a/unicode_constants.h +++ b/unicode_constants.h @@ -21,6 +21,30 @@ * "_TAIL" if instead it represents all but the first byte. This, and * with no additional suffix are both string constants */ +/* +=head1 Unicode Support + +=for apidoc AmU|placeholder|BOM_UTF8 + +This is a macro that evaluates to a string constant of the UTF-8 bytes that +define the Unicode BYTE ORDER MARK (U+FEFF) for the platform that perl +is compiled on. This allows code to use a mnemonic for this character that +works on both ASCII and EBCDIC platforms. +S<C<sizeof(BOM_UTF8) - 1>> can be used to get its length in +bytes. + +=for apidoc AmU|placeholder|REPLACEMENT_CHARACTER_UTF8 + +This is a macro that evaluates to a string constant of the UTF-8 bytes that +define the Unicode REPLACEMENT CHARACTER (U+FFFD) for the platform that perl +is compiled on. This allows code to use a mnemonic for this character that +works on both ASCII and EBCDIC platforms. +S<C<sizeof(REPLACEMENT_CHARACTER_UTF8) - 1>> can be used to get its length in +bytes. + +=cut +*/ + #define UNICODE_MAJOR_VERSION 9 #define UNICODE_DOT_VERSION 0 #define UNICODE_DOT_DOT_VERSION 0 @@ -45,6 +69,10 @@ # define BOM_UTF8_FIRST_BYTE 0xEF /* U+FEFF */ # define BOM_UTF8_TAIL "\xBB\xBF" /* U+FEFF */ +# define BOM_UTF8 "\xEF\xBB\xBF" /* U+FEFF */ + +# define REPLACEMENT_CHARACTER_UTF8 "\xEF\xBF\xBD" /* U+FFFD */ + # define NBSP_NATIVE 0xA0 /* U+00A0 */ # define NBSP_UTF8 "\xC2\xA0" /* U+00A0 */ @@ -84,6 +112,10 @@ # define BOM_UTF8_FIRST_BYTE 0xDD /* U+FEFF */ # define BOM_UTF8_TAIL "\x73\x66\x73" /* U+FEFF */ +# define BOM_UTF8 "\xDD\x73\x66\x73" /* U+FEFF */ + +# define REPLACEMENT_CHARACTER_UTF8 "\xDD\x73\x73\x71" /* U+FFFD */ + # define NBSP_NATIVE 0x41 /* U+00A0 */ # define NBSP_UTF8 "\x80\x41" /* U+00A0 */ @@ -123,6 +155,10 @@ # define BOM_UTF8_FIRST_BYTE 0xDD /* U+FEFF */ # define BOM_UTF8_TAIL "\x72\x65\x72" /* U+FEFF */ +# define BOM_UTF8 "\xDD\x72\x65\x72" /* U+FEFF */ + +# define REPLACEMENT_CHARACTER_UTF8 "\xDD\x72\x72\x70" /* U+FFFD */ + # define NBSP_NATIVE 0x41 /* U+00A0 */ # define NBSP_UTF8 "\x78\x41" /* U+00A0 */ -- Perl5 Master Repository
