Change 13823 by jhi@alpha on 2001/12/21 00:54:49
Make using U+FDD0..U+FDEF (noncharacters since Unicode 3.1),
U+...FFFE, U+...FFFF, and characters beyond U+10FFFF
(the Unicode maximum code point) warnable offenses.
Affected files ...
.... //depot/perl/embed.h#316 edit
.... //depot/perl/embed.pl#306 edit
.... //depot/perl/global.sym#192 edit
.... //depot/perl/op.c#464 edit
.... //depot/perl/pod/perlapi.pod#116 edit
.... //depot/perl/pp.c#325 edit
.... //depot/perl/proto.h#359 edit
.... //depot/perl/t/op/each.t#18 edit
.... //depot/perl/t/op/pat.t#132 edit
.... //depot/perl/t/op/qq.t#2 edit
.... //depot/perl/utf8.c#137 edit
.... //depot/perl/utf8.h#46 edit
Differences ...
==== //depot/perl/embed.h#316 (text+w) ====
Index: perl/embed.h
--- perl/embed.h.~1~ Thu Dec 20 18:00:05 2001
+++ perl/embed.h Thu Dec 20 18:00:05 2001
@@ -755,6 +755,8 @@
#define utf8n_to_uvuni Perl_utf8n_to_uvuni
#define uvchr_to_utf8 Perl_uvchr_to_utf8
#define uvuni_to_utf8 Perl_uvuni_to_utf8
+#define uvchr_to_utf8_flags Perl_uvchr_to_utf8_flags
+#define uvuni_to_utf8_flags Perl_uvuni_to_utf8_flags
#define pv_uni_display Perl_pv_uni_display
#define sv_uni_display Perl_sv_uni_display
#define vivify_defelem Perl_vivify_defelem
@@ -2274,6 +2276,8 @@
#define utf8n_to_uvuni(a,b,c,d) Perl_utf8n_to_uvuni(aTHX_ a,b,c,d)
#define uvchr_to_utf8(a,b) Perl_uvchr_to_utf8(aTHX_ a,b)
#define uvuni_to_utf8(a,b) Perl_uvuni_to_utf8(aTHX_ a,b)
+#define uvchr_to_utf8_flags(a,b,c) Perl_uvchr_to_utf8_flags(aTHX_ a,b,c)
+#define uvuni_to_utf8_flags(a,b,c) Perl_uvuni_to_utf8_flags(aTHX_ a,b,c)
#define pv_uni_display(a,b,c,d,e) Perl_pv_uni_display(aTHX_ a,b,c,d,e)
#define sv_uni_display(a,b,c,d) Perl_sv_uni_display(aTHX_ a,b,c,d)
#define vivify_defelem(a) Perl_vivify_defelem(aTHX_ a)
==== //depot/perl/embed.pl#306 (xtext) ====
Index: perl/embed.pl
--- perl/embed.pl.~1~ Thu Dec 20 18:00:05 2001
+++ perl/embed.pl Thu Dec 20 18:00:05 2001
@@ -1853,7 +1853,9 @@
Adp |UV |utf8n_to_uvchr |U8 *s|STRLEN curlen|STRLEN* retlen|U32 flags
Adp |UV |utf8n_to_uvuni |U8 *s|STRLEN curlen|STRLEN* retlen|U32 flags
Apd |U8* |uvchr_to_utf8 |U8 *d|UV uv
-Apd |U8* |uvuni_to_utf8 |U8 *d|UV uv
+Ap |U8* |uvuni_to_utf8 |U8 *d|UV uv
+Ap |U8* |uvchr_to_utf8_flags |U8 *d|UV uv|UV flags
+Apd |U8* |uvuni_to_utf8_flags |U8 *d|UV uv|UV flags
Apd |char* |pv_uni_display |SV *dsv|U8 *spv|STRLEN len \
|STRLEN pvlim|UV flags
Apd |char* |sv_uni_display |SV *dsv|SV *ssv|STRLEN pvlim|UV flags
==== //depot/perl/global.sym#192 (text+w) ====
Index: perl/global.sym
--- perl/global.sym.~1~ Thu Dec 20 18:00:05 2001
+++ perl/global.sym Thu Dec 20 18:00:05 2001
@@ -157,6 +157,10 @@
Perl_init_stacks
Perl_init_tm
Perl_instr
+Perl_is_lvalue_sub
+Perl_to_uni_upper_lc
+Perl_to_uni_title_lc
+Perl_to_uni_lower_lc
Perl_is_uni_alnum
Perl_is_uni_alnumc
Perl_is_uni_idfirst
@@ -496,6 +500,8 @@
Perl_utf8n_to_uvuni
Perl_uvchr_to_utf8
Perl_uvuni_to_utf8
+Perl_uvchr_to_utf8_flags
+Perl_uvuni_to_utf8_flags
Perl_pv_uni_display
Perl_sv_uni_display
Perl_warn
==== //depot/perl/op.c#464 (text) ====
Index: perl/op.c
--- perl/op.c.~1~ Thu Dec 20 18:00:05 2001
+++ perl/op.c Thu Dec 20 18:00:05 2001
@@ -2866,7 +2866,8 @@
U8 range_mark = UTF_TO_NATIVE(0xff);
sv_catpvn(transv, (char *)&range_mark, 1);
}
- t = uvuni_to_utf8(tmpbuf, 0x7fffffff);
+ t = uvuni_to_utf8_flags(tmpbuf, 0x7fffffff,
+ UNICODE_ALLOW_SUPER);
sv_catpvn(transv, (char*)tmpbuf, t - tmpbuf);
t = (U8*)SvPVX(transv);
tlen = SvCUR(transv);
==== //depot/perl/pod/perlapi.pod#116 (text+w) ====
Index: perl/pod/perlapi.pod
--- perl/pod/perlapi.pod.~1~ Thu Dec 20 18:00:05 2001
+++ perl/pod/perlapi.pod Thu Dec 20 18:00:05 2001
@@ -1573,8 +1573,8 @@
Returns a pointer to the next character after the parsed
vstring, as well as updating the passed in sv.
- *
-Function must be called like
+ *
+Function must be called like
sv = NEWSV(92,5);
s = new_vstring(s,sv);
@@ -4453,20 +4453,28 @@
=for hackers
Found in file utf8.c
-=item uvuni_to_utf8
+=item uvuni_to_utf8_flags
Adds the UTF8 representation of the Unicode codepoint C<uv> to the end
of the string C<d>; C<d> should be have at least C<UTF8_MAXLEN+1> free
bytes available. The return value is the pointer to the byte after the
end of the new character. In other words,
+ d = uvuni_to_utf8_flags(d, uv, flags);
+
+or, in most cases,
+
d = uvuni_to_utf8(d, uv);
+(which is equivalent to)
+
+ d = uvuni_to_utf8_flags(d, uv, 0);
+
is the recommended Unicode-aware way of saying
*(d++) = uv;
- U8* uvuni_to_utf8(U8 *d, UV uv)
+ U8* uvuni_to_utf8_flags(U8 *d, UV uv, UV flags)
=for hackers
Found in file utf8.c
==== //depot/perl/pp.c#325 (text) ====
Index: perl/pp.c
--- perl/pp.c.~1~ Thu Dec 20 18:00:05 2001
+++ perl/pp.c Thu Dec 20 18:00:05 2001
@@ -2258,7 +2258,7 @@
while (tmps < send) {
UV c = utf8n_to_uvchr(tmps, send-tmps, &l, UTF8_ALLOW_ANYUV);
tmps += UTF8SKIP(tmps);
- result = uvchr_to_utf8(result, ~c);
+ result = uvchr_to_utf8_flags(result, ~c, UNICODE_ALLOW_ANY);
}
*result = '\0';
result -= targlen;
@@ -3148,7 +3148,8 @@
if (value > 255 && !IN_BYTES) {
SvGROW(TARG, UNISKIP(value)+1);
- tmps = (char*)uvchr_to_utf8((U8*)SvPVX(TARG), value);
+ tmps = (char*)uvchr_to_utf8_flags((U8*)SvPVX(TARG), value,
+ UNICODE_ALLOW_SUPER);
SvCUR_set(TARG, tmps - SvPVX(TARG));
*tmps = '\0';
(void)SvPOK_only(TARG);
==== //depot/perl/proto.h#359 (text+w) ====
Index: perl/proto.h
--- perl/proto.h.~1~ Thu Dec 20 18:00:05 2001
+++ perl/proto.h Thu Dec 20 18:00:05 2001
@@ -832,6 +832,8 @@
PERL_CALLCONV UV Perl_utf8n_to_uvuni(pTHX_ U8 *s, STRLEN curlen, STRLEN*
retlen, U32 flags);
PERL_CALLCONV U8* Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv);
PERL_CALLCONV U8* Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv);
+PERL_CALLCONV U8* Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags);
+PERL_CALLCONV U8* Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags);
PERL_CALLCONV char* Perl_pv_uni_display(pTHX_ SV *dsv, U8 *spv, STRLEN len, STRLEN
pvlim, UV flags);
PERL_CALLCONV char* Perl_sv_uni_display(pTHX_ SV *dsv, SV *ssv, STRLEN pvlim, UV
flags);
PERL_CALLCONV void Perl_vivify_defelem(pTHX_ SV* sv);
==== //depot/perl/t/op/each.t#18 (xtext) ====
Index: perl/t/op/each.t
--- perl/t/op/each.t.~1~ Thu Dec 20 18:00:05 2001
+++ perl/t/op/each.t Thu Dec 20 18:00:05 2001
@@ -135,7 +135,7 @@
# Check for Unicode hash keys.
%u = ("\x{12}", "f", "\x{123}", "fo", "\x{1234}", "foo");
$u{"\x{12345}"} = "bar";
-@u{"\x{123456}"} = "zap";
+@u{"\x{10FFFD}"} = "zap";
my %u2;
foreach (keys %u) {
==== //depot/perl/t/op/pat.t#132 (xtext) ====
Index: perl/t/op/pat.t
--- perl/t/op/pat.t.~1~ Thu Dec 20 18:00:05 2001
+++ perl/t/op/pat.t Thu Dec 20 18:00:05 2001
@@ -1618,9 +1618,9 @@
{
# from Robin Houston
- my $x = "\x{12345678}";
+ my $x = "\x{10FFFD}";
$x =~ s/(.)/$1/g;
- print "not " unless ord($x) == 0x12345678 && length($x) == 1;
+ print "not " unless ord($x) == 0x10FFFD && length($x) == 1;
print "ok 587\n";
}
==== //depot/perl/t/op/qq.t#2 (text) ====
Index: perl/t/op/qq.t
--- perl/t/op/qq.t.~1~ Thu Dec 20 18:00:05 2001
+++ perl/t/op/qq.t Thu Dec 20 18:00:05 2001
@@ -60,4 +60,4 @@
chr 114);
is ("\x{0_06_5}", chr 101);
is ("\x{1234}", chr 4660);
-is ("\x{98765432}", chr 2557891634);
+is ("\x{10FFFD}", chr 1114109);
==== //depot/perl/utf8.c#137 (text) ====
Index: perl/utf8.c
--- perl/utf8.c.~1~ Thu Dec 20 18:00:05 2001
+++ perl/utf8.c Thu Dec 20 18:00:05 2001
@@ -27,15 +27,23 @@
/* Unicode support */
/*
-=for apidoc A|U8 *|uvuni_to_utf8|U8 *d|UV uv
+=for apidoc A|U8 *|uvuni_to_utf8_flags|U8 *d|UV uv|UV flags
Adds the UTF8 representation of the Unicode codepoint C<uv> to the end
of the string C<d>; C<d> should be have at least C<UTF8_MAXLEN+1> free
bytes available. The return value is the pointer to the byte after the
end of the new character. In other words,
+ d = uvuni_to_utf8_flags(d, uv, flags);
+
+or, in most cases,
+
d = uvuni_to_utf8(d, uv);
+(which is equivalent to)
+
+ d = uvuni_to_utf8_flags(d, uv, 0);
+
is the recommended Unicode-aware way of saying
*(d++) = uv;
@@ -44,13 +52,26 @@
*/
U8 *
-Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv)
+Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags)
{
if (ckWARN_d(WARN_UTF8)) {
- if (UNICODE_IS_SURROGATE(uv))
+ if (UNICODE_IS_SURROGATE(uv) &&
+ !(flags & UNICODE_ALLOW_SURROGATE))
Perl_warner(aTHX_ WARN_UTF8, "UTF-16 surrogate 0x%04"UVxf, uv);
- else if ((uv >= 0xFDD0 && uv <= 0xFDEF) ||
- (uv == 0xFFFE || uv == 0xFFFF))
+ else if (
+ ((uv >= 0xFDD0 && uv <= 0xFDEF &&
+ !(flags & UNICODE_ALLOW_FDD0))
+ ||
+ ((uv & 0xFFFF) == 0xFFFE &&
+ !(flags & UNICODE_ALLOW_FFFE))
+ ||
+ ((uv & 0xFFFF) == 0xFFFF &&
+ !(flags & UNICODE_ALLOW_FFFF))) &&
+ /* UNICODE_ALLOW_SUPER includes
+ * FFFEs and FFFFs beyond 0x10FFFF. */
+ ((uv <= PERL_UNICODE_MAX) ||
+ !(flags & UNICODE_ALLOW_SUPER))
+ )
Perl_warner(aTHX_ WARN_UTF8,
"Unicode character 0x%04"UVxf" is illegal", uv);
}
@@ -138,9 +159,14 @@
#endif
#endif /* Loop style */
}
+
+U8 *
+Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv)
+{
+ return Perl_uvuni_to_utf8_flags(aTHX_ d, uv, 0);
+}
-
/*
=for apidoc A|STRLEN|is_utf8_char|U8 *s
@@ -1544,9 +1570,14 @@
U8 *
Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv)
{
- return Perl_uvuni_to_utf8(aTHX_ d, NATIVE_TO_UNI(uv));
+ return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), 0);
}
+U8 *
+Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags)
+{
+ return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), flags);
+}
/*
=for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags
==== //depot/perl/utf8.h#46 (text) ====
Index: perl/utf8.h
--- perl/utf8.h.~1~ Thu Dec 20 18:00:05 2001
+++ perl/utf8.h Thu Dec 20 18:00:05 2001
@@ -166,6 +166,17 @@
#define UNICODE_BYTER_ORDER_MARK 0xfffe
#define UNICODE_ILLEGAL 0xffff
+/* Though our UTF-8 encoding can go beyond this,
+ * let's be conservative. */
+#define PERL_UNICODE_MAX 0x10FFFF
+
+#define UNICODE_ALLOW_SURROGATE 0x0001 /* Allow UTF-16 surrogates (EVIL) */
+#define UNICODE_ALLOW_FDD0 0x0002 /* Allow the U+FDD0...U+FDEF */
+#define UNICODE_ALLOW_FFFE 0x0004 /* Allow 0xFFFE, 0x1FFFE, ... */
+#define UNICODE_ALLOW_FFFF 0x0008 /* Allow 0xFFFE, 0x1FFFE, ... */
+#define UNICODE_ALLOW_SUPER 0x0010 /* Allow past 10xFFFF */
+#define UNICODE_ALLOW_ANY 0xFFFF
+
#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \
(c) <= UNICODE_SURROGATE_LAST)
#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACEMENT)
End of Patch.
