Commit: 177dfc6384b926dd19e3b7e98a995ccb4da9167c Author: mano-wii Date: Fri Nov 22 12:26:54 2019 -0300 Branches: master https://developer.blender.org/rB177dfc6384b926dd19e3b7e98a995ccb4da9167c
Fix T71273: Bad encoding of utf-8 for Text objects `BLI_strncpy_wchar_from_utf8` internally assumes `wchar_t` is 32 bits which is not the case on windows. The solution is to replace `wchar_t` with `char32_t`. Thanks to @robbott for compatibility on macOS. Differential Revision: https://developer.blender.org/D6198 =================================================================== M extern/wcwidth/README.blender M extern/wcwidth/wcwidth.c M extern/wcwidth/wcwidth.h M source/blender/blenfont/intern/blf_font.c M source/blender/blenkernel/BKE_font.h M source/blender/blenkernel/intern/font.c M source/blender/blenkernel/intern/object_dupli.c M source/blender/blenkernel/intern/text.c M source/blender/blenlib/BLI_string_cursor_utf8.h M source/blender/blenlib/BLI_string_utf8.h M source/blender/blenlib/BLI_sys_types.h M source/blender/blenlib/intern/string_cursor_utf8.c M source/blender/blenlib/intern/string_utf8.c M source/blender/editors/curve/editfont.c M source/blender/editors/curve/editfont_undo.c M source/blender/makesrna/intern/rna_curve.c =================================================================== diff --git a/extern/wcwidth/README.blender b/extern/wcwidth/README.blender index 27c8574d1d7..3b32ddc5bc9 100644 --- a/extern/wcwidth/README.blender +++ b/extern/wcwidth/README.blender @@ -2,4 +2,6 @@ Project: WC Width URL: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c License: ICS Upstream version: 2007-05-26 -Local modifications: None +Local modifications: +* Fix T33192 + Bad encoding of utf-8 on windows systems. diff --git a/extern/wcwidth/wcwidth.c b/extern/wcwidth/wcwidth.c index 8cab81e32cd..8922ffe0842 100644 --- a/extern/wcwidth/wcwidth.c +++ b/extern/wcwidth/wcwidth.c @@ -59,8 +59,6 @@ * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c */ -#include <wchar.h> - #include "wcwidth.h" struct interval { @@ -69,7 +67,7 @@ struct interval { }; /* auxiliary function for binary search in interval table */ -static int bisearch(wchar_t ucs, const struct interval *table, int max) { +static int bisearch(char32_t ucs, const struct interval *table, int max) { int min = 0; int mid; @@ -121,7 +119,7 @@ static int bisearch(wchar_t ucs, const struct interval *table, int max) { * in ISO 10646. */ -int mk_wcwidth(wchar_t ucs) +int mk_wcwidth(char32_t ucs) { /* sorted list of non-overlapping intervals of non-spacing characters */ /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ @@ -206,7 +204,7 @@ int mk_wcwidth(wchar_t ucs) } -int mk_wcswidth(const wchar_t *pwcs, size_t n) +int mk_wcswidth(const char32_t *pwcs, size_t n) { int w, width = 0; @@ -229,7 +227,7 @@ int mk_wcswidth(const wchar_t *pwcs, size_t n) * the traditional terminal character-width behaviour. It is not * otherwise recommended for general use. */ -int mk_wcwidth_cjk(wchar_t ucs) +int mk_wcwidth_cjk(char32_t ucs) { /* sorted list of non-overlapping intervals of East Asian Ambiguous * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */ @@ -297,7 +295,7 @@ int mk_wcwidth_cjk(wchar_t ucs) } -int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n) +int mk_wcswidth_cjk(const char32_t *pwcs, size_t n) { int w, width = 0; diff --git a/extern/wcwidth/wcwidth.h b/extern/wcwidth/wcwidth.h index 9ae6ec9ef14..d87eaf20695 100644 --- a/extern/wcwidth/wcwidth.h +++ b/extern/wcwidth/wcwidth.h @@ -20,11 +20,19 @@ #ifndef __WCWIDTH_H__ #define __WCWIDTH_H__ -#include <wchar.h> +#ifndef __cplusplus +# if defined(__APPLE__) +/* The <uchar.h> standard header is missing on macOS. */ +#include <stddef.h> +typedef unsigned int char32_t; +# else +# include <uchar.h> +# endif +#endif -int mk_wcwidth(wchar_t ucs); -int mk_wcswidth(const wchar_t *pwcs, size_t n); -int mk_wcwidth_cjk(wchar_t ucs); -int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n); +int mk_wcwidth(char32_t ucs); +int mk_wcswidth(const char32_t *pwcs, size_t n); +int mk_wcwidth_cjk(char32_t ucs); +int mk_wcswidth_cjk(const char32_t *pwcs, size_t n); #endif diff --git a/source/blender/blenfont/intern/blf_font.c b/source/blender/blenfont/intern/blf_font.c index e0dfa6a2223..1c06dfd3f70 100644 --- a/source/blender/blenfont/intern/blf_font.c +++ b/source/blender/blenfont/intern/blf_font.c @@ -504,7 +504,7 @@ int blf_font_draw_mono(FontBLF *font, const char *str, size_t len, int cwidth) /* do not return this loop if clipped, we want every character tested */ blf_glyph_render(font, gc, g, (float)pen_x, (float)pen_y); - col = BLI_wcwidth((wchar_t)c); + col = BLI_wcwidth((char32_t)c); if (col < 0) { col = 1; } diff --git a/source/blender/blenkernel/BKE_font.h b/source/blender/blenkernel/BKE_font.h index 3dd2a551a93..1780ac9fc3e 100644 --- a/source/blender/blenkernel/BKE_font.h +++ b/source/blender/blenkernel/BKE_font.h @@ -27,8 +27,6 @@ extern "C" { #endif -#include <wchar.h> - struct CharInfo; struct Curve; struct Main; @@ -48,7 +46,7 @@ typedef struct EditFontSelBox { } EditFontSelBox; typedef struct EditFont { - wchar_t *textbuf; + char32_t *textbuf; struct CharInfo *textbufinfo; /* array of rectangles & rotation */ @@ -90,7 +88,7 @@ bool BKE_vfont_to_curve_ex(struct Object *ob, struct Curve *cu, int mode, struct ListBase *r_nubase, - const wchar_t **r_text, + const char32_t **r_text, int *r_text_len, bool *r_text_free, struct CharTrans **r_chartransdata); @@ -101,13 +99,13 @@ int BKE_vfont_select_get(struct Object *ob, int *r_start, int *r_end); void BKE_vfont_select_clamp(struct Object *ob); void BKE_vfont_clipboard_free(void); -void BKE_vfont_clipboard_set(const wchar_t *text_buf, +void BKE_vfont_clipboard_set(const char32_t *text_buf, const struct CharInfo *info_buf, const size_t len); -void BKE_vfont_clipboard_get(wchar_t **r_text_buf, +void BKE_vfont_clipboard_get(char32_t **r_text_buf, struct CharInfo **r_info_buf, size_t *r_len_utf8, - size_t *r_len_wchar); + size_t *r_len_utf32); #ifdef __cplusplus } diff --git a/source/blender/blenkernel/intern/font.c b/source/blender/blenkernel/intern/font.c index 250e6ff6a7b..aac3e8cc7fc 100644 --- a/source/blender/blenkernel/intern/font.c +++ b/source/blender/blenkernel/intern/font.c @@ -25,7 +25,6 @@ #include <string.h> #include <math.h> #include <stdlib.h> -#include <wchar.h> #include <wctype.h> #include "CLG_log.h" @@ -691,7 +690,7 @@ static bool vfont_to_curve(Object *ob, int mode, VFontToCurveIter *iter_data, ListBase *r_nubase, - const wchar_t **r_text, + const char32_t **r_text, int *r_text_len, bool *r_text_free, struct CharTrans **r_chartransdata) @@ -712,8 +711,8 @@ static bool vfont_to_curve(Object *ob, int curbox; int selstart, selend; int cnr = 0, lnr = 0, wsnr = 0; - const wchar_t *mem = NULL; - wchar_t ascii; + const char32_t *mem = NULL; + char32_t ascii; bool ok = false; const float font_size = cu->fsize * iter_data->scale_to_fit; const float xof_scale = cu->xof / font_size; @@ -759,16 +758,16 @@ static bool vfont_to_curve(Object *ob, custrinfo = ef->textbufinfo; } else { - wchar_t *mem_tmp; + char32_t *mem_tmp; slen = cu->len_wchar; /* Create unicode string */ - mem_tmp = MEM_malloc_arrayN((slen + 1), sizeof(wchar_t), "convertedmem"); + mem_tmp = MEM_malloc_arrayN((slen + 1), sizeof(*mem_tmp), "convertedmem"); if (!mem_tmp) { return ok; } - BLI_strncpy_wchar_from_utf8(mem_tmp, cu->str, slen + 1); + BLI_str_utf8_as_utf32(mem_tmp, cu->str, slen + 1); if (cu->strinfo == NULL) { /* old file */ cu->strinfo = MEM_calloc_arrayN((slen + 4), sizeof(CharInfo), "strinfo compat"); @@ -1605,7 +1604,7 @@ bool BKE_vfont_to_curve_ex(Object *ob, Curve *cu, int mode, ListBase *r_nubase, - const wchar_t **r_text, + const char32_t **r_text, int *r_text_len, bool *r_text_free, struct CharTrans **r_chartransdata) @@ -1649,9 +1648,9 @@ bool BKE_vfont_to_curve(Object *ob, int mode) * \{ */ static struct { - wchar_t *text_buffer; + char32_t *text_buffer; CharInfo *info_buffer; - size_t len_wchar; + size_t len_utf32; size_t len_utf8; } g_vfont_clipboard = {NULL}; @@ -1659,19 +1658,19 @@ void BKE_vfont_clipboard_free(void) { MEM_SAFE_FREE(g_vfont_clipboard.text_buffer); MEM_SAFE_FREE(g_vfont_clipboard.info_buffer); - g_vfont_clipboard.len_wchar = 0; + g_vfont_clipboard.len_utf32 = 0; g_vfont_clipboard.len_utf8 = 0; } -void BKE_vfont_clipboard_set(const wchar_t *text_buf, const CharInfo *info_buf, const size_t len) +void BKE_vfont_clipboard_set(const char32_t *text_buf, const CharInfo *info_buf, const size_t len) { - wchar_t *text; + char32_t *text; CharInfo *info; /* clean previous buffers*/ BKE_vfont_clipboard_free(); - text = MEM_malloc_arrayN((len + 1), sizeof(wchar_t), __func__); + text = MEM_malloc_arrayN((len + 1), sizeof(*text), __func__); if (text == NULL) { return; } @@ -1682,21 +1681,21 @@ void BKE_vfont_clipboard_set(const wchar_t *text_buf, const CharInfo *info_buf, return; } - memcpy(text, text_buf, len * sizeof(wchar_t)); + memcpy(text, text_buf, len * sizeof(*text)); text[len] = '\0'; memcpy(info, info_buf, len * sizeof(CharInfo)); /* store new buffers */ g_vfont_clipboard.text_buffer = text; g_vfont_clipboard.info_buffer = info; - g_vfont_clipboard.len_utf8 = BLI_wstrlen_utf8(text); - g_vfont_clipboard.len_wchar = len; + g_vfont_clipboard.len_utf8 = BLI_str_utf32_as_utf8_len(text); + g_vfont_clipboard.len_utf32 = len; } -void BKE_vfont_clipboard_get(wchar_t **r_text_buf, +void BKE_vfont_clipboard_get(char32_t **r_text_buf, CharInfo **r_info_buf, size_t *r_len_utf8, - size_t *r_len_wchar) + size_t *r_len_utf32) { if (r_text_buf) { *r_text_buf = g_vfont_clipboard.text_buffer; @@ -1706,8 +1705,8 @@ void BKE_vfont_clipboard_get(wchar_t **r_text_buf, *r_info_buf = g_vfont_clipboard.info_buffer; } - if (r_len_wchar) { - *r_len_wchar = g_vfont_clipboard.len_wchar; + if (r_len_utf32) { + *r_len_utf32 = g_vfont_clipboard.len_utf32; } if (r_len_utf8) { diff --git a/source/blender/blenkernel/intern/object_dupli.c b/source/blender/blenkernel/intern/object_dupli.c index 6b6c68b197e..eceeb231285 100644 --- a/source/blender/blenkernel/intern/object_dupli.c +++ b/source/blender/blenkernel/intern/object_dupli.c @@ -491,7 +491,7 @@ static void make_dupli @@ Diff output truncated at 10240 characters. @@ _______________________________________________ Bf-blender-cvs mailing list Bf-blender-cvs@blender.org https://lists.blender.org/mailman/listinfo/bf-blender-cvs