On Windows, lilypond can't treat unicode filenames. On linux, it can (UTF-8 unicode filenames).
So, I'm trying to add a unicode filename support for Windows lilypond. I attach the patch. It replaces main(), and hooks filename related functions. This converts between UTF-16 unicode (Windows) and UTF-8 unicode (lilypond, libguile etc.). As a result, lilypond can treat unicode filenames of *.ly, *.mid, *.ps. However, it can't treat *.pdf, yet. Ghostscript-8.70 that are included in the binary distribution of lilypond can't treat the unicode filenames. Ghostscript-9.10 or later can treat the unicode filenames.
>From bb35555f8bd6bcaf41de57e8981ac57a3ac0ef8c Mon Sep 17 00:00:00 2001 From: Masamichi Hosoda <truer...@trueroad.jp> Date: Sat, 7 Mar 2015 18:42:57 +0900 Subject: [PATCH] Add unicode filename support for Windows --- flower/include/mingw-utf8-conv.hh | 7 ++ flower/include/mingw-utf8-func.hh | 16 ++++ flower/include/mingw-utf8-hook.hh | 6 ++ flower/include/mingw-utf8.hh | 11 +++ flower/mingw-utf8-conv.cc | 58 ++++++++++++ flower/mingw-utf8-func.cc | 157 +++++++++++++++++++++++++++++++++ flower/mingw-utf8-hook.cc | 179 ++++++++++++++++++++++++++++++++++++++ flower/mingw-utf8-main.cc | 84 ++++++++++++++++++ lily/GNUmakefile | 1 + lily/main.cc | 4 + 10 files changed, 523 insertions(+) create mode 100644 flower/include/mingw-utf8-conv.hh create mode 100644 flower/include/mingw-utf8-func.hh create mode 100644 flower/include/mingw-utf8-hook.hh create mode 100644 flower/include/mingw-utf8.hh create mode 100644 flower/mingw-utf8-conv.cc create mode 100644 flower/mingw-utf8-func.cc create mode 100644 flower/mingw-utf8-hook.cc create mode 100644 flower/mingw-utf8-main.cc diff --git a/flower/include/mingw-utf8-conv.hh b/flower/include/mingw-utf8-conv.hh new file mode 100644 index 0000000..1cabe21 --- /dev/null +++ b/flower/include/mingw-utf8-conv.hh @@ -0,0 +1,7 @@ +#ifndef __MINGW_UTF8_CONV_H__ +#define __MINGW_UTF8_CONV_H__ + +std::vector<char> mingw_utf8_16to8 (const wchar_t *wc); +std::vector<wchar_t> mingw_utf8_8to16 (const char *c); + +#endif // __MINGW_UTF8_CONV_H__ diff --git a/flower/include/mingw-utf8-func.hh b/flower/include/mingw-utf8-func.hh new file mode 100644 index 0000000..d4a0a29 --- /dev/null +++ b/flower/include/mingw-utf8-func.hh @@ -0,0 +1,16 @@ +#ifndef __MINGW_UTF8_FUNC_H__ +#define __MINGW_UTF8_FUNC_H__ + +FILE *utf8_fopen(const char *path, const char *mode); +FILE *utf8_freopen (const char *path, const char *mode, FILE *stream); +int utf8__stat (const char *path, struct _stat *buff); +char *utf8_getcwd(char *buff, int size); +char *utf8_getenv(const char *var); +int utf8_open (const char *path, int flag, ...); +int utf8_system (const char *command); +int utf8__unlink (const char *path); +gchar *utf8_g_locale_to_from_utf8 (const gchar *string, gssize len, + gsize *bytes_read, gssize *bytes_written, + GError **error); + +#endif // __MINGW_UTF8_FUNC_H__ diff --git a/flower/include/mingw-utf8-hook.hh b/flower/include/mingw-utf8-hook.hh new file mode 100644 index 0000000..0eea976 --- /dev/null +++ b/flower/include/mingw-utf8-hook.hh @@ -0,0 +1,6 @@ +#ifndef __MINGW_UTF8_HOOK_H__ +#define __MINGW_UTF8_HOOK_H__ + +void mingw_utf8_hook(void); + +#endif // __MINGW_UTF8_HOOK_H__ diff --git a/flower/include/mingw-utf8.hh b/flower/include/mingw-utf8.hh new file mode 100644 index 0000000..feb2fab --- /dev/null +++ b/flower/include/mingw-utf8.hh @@ -0,0 +1,11 @@ +#ifdef __MINGW32__ + +#ifndef __MINGW_UTF8_H__ +#define __MINGW_UTF8_H__ + +#define main(...) utf8_main(__VA_ARGS__) + +int utf8_main(int argc, char **argv, char **envp); + +#endif // __MINGW_UTF8_H__ +#endif // __MINGW32__ diff --git a/flower/mingw-utf8-conv.cc b/flower/mingw-utf8-conv.cc new file mode 100644 index 0000000..fa04da9 --- /dev/null +++ b/flower/mingw-utf8-conv.cc @@ -0,0 +1,58 @@ +/* + This file is part of LilyPond, the GNU music typesetter. + + Copyright (C) 2015 Masamichi Hosoda <truer...@trueroad.jp> + + LilyPond is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + LilyPond is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LilyPond. If not, see <http://www.gnu.org/licenses/>. +*/ + +#ifdef __MINGW32__ + +#include <cwchar> +#include <vector> +#include <windows.h> + +#include "mingw-utf8-conv.hh" + +// Convert from UTF-16 to UTF-8 +std::vector<char> mingw_utf8_16to8 (const wchar_t *wc) +{ + int size = WideCharToMultiByte (CP_UTF8, 0, wc, -1, NULL, 0, NULL, NULL); + std::vector<char> retval (size); + if (size) + { + WideCharToMultiByte (CP_UTF8, 0, wc, -1, + retval.data(), retval.size(), NULL, NULL); + } + else + retval.push_back ('\0'); + return retval; +} + +// Convert from UTF-8 to UTF-16 +std::vector<wchar_t> mingw_utf8_8to16 (const char *c) +{ + int size = MultiByteToWideChar (CP_UTF8, 0, c, -1, NULL, 0); + std::vector<wchar_t> retval (size); + if (size) + { + MultiByteToWideChar (CP_UTF8, 0, c, -1, + retval.data(), retval.size()); + } + else + retval.push_back (L'\0'); + return retval; +} + +#endif // __MINGW32__ diff --git a/flower/mingw-utf8-func.cc b/flower/mingw-utf8-func.cc new file mode 100644 index 0000000..b19ead4 --- /dev/null +++ b/flower/mingw-utf8-func.cc @@ -0,0 +1,157 @@ +/* + This file is part of LilyPond, the GNU music typesetter. + + Copyright (C) 2015 Masamichi Hosoda <truer...@trueroad.jp> + + LilyPond is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + LilyPond is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LilyPond. If not, see <http://www.gnu.org/licenses/>. +*/ + +#ifdef __MINGW32__ + +#include <cwchar> +#include <cstdio> +#include <cstdlib> +#include <cstdarg> +#include <vector> +#include <sys/stat.h> +#include <fcntl.h> +#include <glib.h> + +#include "mingw-utf8-func.hh" +#include "mingw-utf8-conv.hh" + +// msvcrt fopen() +FILE *utf8_fopen (const char *path, const char *mode) +{ + //printf("hook!! fopen (\"%s\", \"%s\")\n", path, mode); + + std::vector<wchar_t> wpath (mingw_utf8_8to16 (path)); + std::vector<wchar_t> wmode (mingw_utf8_8to16 (mode)); + return _wfopen (wpath.data(), wmode.data()); +} + +// msvcrt freopen() +FILE *utf8_freopen (const char *path, const char *mode, FILE *stream) +{ + //printf("hook!! freopen (\"%s\", \"%s\", stream)\n", path, mode); + + std::vector<wchar_t> wpath (mingw_utf8_8to16 (path)); + std::vector<wchar_t> wmode (mingw_utf8_8to16 (mode)); + return _wfreopen (wpath.data(), wmode.data(), stream); +} + +// msvcrt _stat() / stat() +int utf8__stat (const char *path, struct _stat *buff) +{ + //printf("hook!! _stat (\"%s\", buff)\n", path); + + std::vector<wchar_t> wpath (mingw_utf8_8to16 (path)); + return _wstat (wpath.data(), buff); +} + +// msvcrt getcwd() / _getcwd() +char *utf8_getcwd(char *buff, int size) +{ + //printf("hook!! getcwd (0x%p, %d)\n", buff, size); + + wchar_t *wp = _wgetcwd(NULL, 0); + std::vector<char> p (mingw_utf8_16to8 (wp)); + free (wp); + + if (buff == NULL) + buff = (char*) malloc (p.size()); + strncpy(buff, p.data(), size); + buff[size]=0; + return buff; +} + +// msvcrt getenv() +char *utf8_getenv(const char *var) +{ + //fputs("hook!! getenv (\"", stdout); + //fputs(var, stdout); + //fputs("\")\n", stdout); + + static std::vector<char> env_ret; + std::vector<wchar_t> wvar (mingw_utf8_8to16 (var)); + wchar_t *e = _wgetenv(wvar.data()); + if (e == NULL) + return NULL; + env_ret = mingw_utf8_16to8 (e); + return env_ret.data(); +} + +// msvcrt open() / _open() +int utf8_open (const char *path, int flag, ...) +{ + //printf("hook!! open (\"%s\", flag, ...)\n", path); + + int mode = 0777; + va_list list; + va_start (list, flag); + if ( flag & O_CREAT ) + mode = va_arg (list, int); + va_end (list); + + std::vector<wchar_t> wpath (mingw_utf8_8to16 (path)); + return _wopen(wpath.data(), flag, mode); +} + +// msvcrt system() +int utf8_system (const char *command) +{ + //printf("hook!! system (\"%s\")\n", command); + + std::vector<wchar_t> wcommand (mingw_utf8_8to16 (command)); + return _wsystem(wcommand.data()); +} + +// msvcrt _unlink() / unlink() +int utf8__unlink (const char *path) +{ + //printf("hook!! _unlink (\"%s\")\n", path); + + std::vector<wchar_t> wpath (mingw_utf8_8to16 (path)); + return _wunlink(wpath.data()); +} + +// glib g_locale_{to|from}_utf8() +// from gconvert.c g_locale_{to|from}_utf8() and strdup_len() +gchar *utf8_g_locale_to_from_utf8 (const gchar *string, gssize len, + gsize *bytes_read, gssize *bytes_written, + GError **error) +{ + //printf("hook!! g_locale_{to|from}_utf8\n"); + + gsize real_len; + + if (len < 0) + real_len = strlen (string); + else + { + real_len = 0; + + while (real_len < len && string[real_len]) + real_len++; + } + + if (bytes_read) + *bytes_read = real_len; + if (bytes_written) + *bytes_written = real_len; + + return g_strndup (string, real_len); +} + +#endif // __MINGW32__ diff --git a/flower/mingw-utf8-hook.cc b/flower/mingw-utf8-hook.cc new file mode 100644 index 0000000..ddf73f1 --- /dev/null +++ b/flower/mingw-utf8-hook.cc @@ -0,0 +1,179 @@ +/* + This file is part of LilyPond, the GNU music typesetter. + + Copyright (C) 2015 Masamichi Hosoda <truer...@trueroad.jp> + + LilyPond is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + LilyPond is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LilyPond. If not, see <http://www.gnu.org/licenses/>. +*/ + +#ifdef __MINGW32__ + +#include <cwchar> +#include <vector> +#include <windows.h> +#include <tlhelp32.h> +#include <dbghelp.h> +#include <glib.h> + +//#include <cstdio> + +#include "mingw-utf8-hook.hh" +#include "mingw-utf8-func.hh" + +// Internal use +namespace +{ + void hook_all (PROC func_old, PROC func_new); + void hook_one (PROC func_old, PROC func_new, + HMODULE hmod, LPBYTE pbase); +} + +// Address of hook target functions. +extern "C" +{ + extern PROC _imp__fopen; + extern PROC _imp__freopen; + extern PROC _imp___stat; + extern PROC _imp__stat; + extern PROC _imp__getcwd; + extern PROC _imp___getcwd; + extern PROC _imp__getenv; + extern PROC _imp__open; + extern PROC _imp___open; + extern PROC _imp__system; + extern PROC _imp___unlink; + extern PROC _imp__unlink; + extern PROC _imp__g_locale_to_utf8; + extern PROC _imp__g_locale_from_utf8; +} + +// Hook filename related functions. +void mingw_utf8_hook (void) +{ + //printf("try to hook fopen\n"); + hook_all (_imp__fopen, (PROC)utf8_fopen); + //printf("try to hook freopen\n"); + hook_all (_imp__freopen, (PROC)utf8_freopen); + //printf("try to hook _stat\n"); + hook_all (_imp___stat, (PROC)utf8__stat); + //printf("try to hook stat\n"); + hook_all (_imp__stat, (PROC)utf8__stat); + //printf("try to hook getcwd\n"); + hook_all (_imp__getcwd, (PROC)utf8_getcwd); + //printf("try to hook _getcwd\n"); + hook_all (_imp___getcwd, (PROC)utf8_getcwd); + //printf("try to hook getenv\n"); + hook_all (_imp__getenv, (PROC)utf8_getenv); + //printf("try to hook open\n"); + hook_all (_imp__open, (PROC)utf8_open); + //printf("try to hook _open\n"); + hook_all (_imp___open, (PROC)utf8_open); + //printf("try to hook system\n"); + hook_all (_imp__system, (PROC)utf8_system); + //printf("try to hook _unlink\n"); + hook_all (_imp___unlink, (PROC)utf8__unlink); + //printf("try to hook unlink\n"); + hook_all (_imp__unlink, (PROC)utf8__unlink); + //printf("try to hook g_locale_to_utf8\n"); + hook_all (_imp__g_locale_to_utf8, (PROC)utf8_g_locale_to_from_utf8); + //printf("try to hook g_locale_from_utf8\n"); + hook_all (_imp__g_locale_from_utf8, (PROC)utf8_g_locale_to_from_utf8); +} + +namespace +{ + // Hook all loaded modules. + void hook_all (PROC func_old, PROC func_new) + { + //printf ("hook_all enter old %p, new %p\n", func_old, func_new); + //fflush (stdout); + + // Take a snapshot of modules. + HANDLE hmod_snap = + CreateToolhelp32Snapshot (TH32CS_SNAPMODULE, GetCurrentProcessId ()); + if (hmod_snap == INVALID_HANDLE_VALUE) + return; + + // Enumerate snapshotted modules. + MODULEENTRY32W me; + me.dwSize = sizeof (me); + BOOL bresult = Module32FirstW (hmod_snap, &me); + + while (bresult) + { + //printf (" Module %S...\n", me.szModule); + //fflush (stdout); + + // Hook one module. + hook_one(func_old, func_new, me.hModule, me.modBaseAddr); + bresult = Module32NextW (hmod_snap, &me); + } + + //printf("hook_all exit\n"); + //fflush(stdout); + } + + // Hook one module. + void hook_one (PROC func_old, PROC func_new, + HMODULE hmod, LPBYTE pbase) + { + ULONG size; + + // Get import descriptor. + PIMAGE_IMPORT_DESCRIPTOR pimp_desc = (PIMAGE_IMPORT_DESCRIPTOR) + ImageDirectoryEntryToData (hmod, TRUE, + IMAGE_DIRECTORY_ENTRY_IMPORT, &size); + if (pimp_desc == NULL) + return; + + // Enumerate import modules. + while (pimp_desc->Name) + { + // Get import address table. + PIMAGE_THUNK_DATA pthunk = (PIMAGE_THUNK_DATA) + (pbase + pimp_desc->FirstThunk); + + // Enumerate import address table. + while (pthunk->u1.Function) + { + // Get address of imported function. + PROC *p = (PROC*) &pthunk->u1.Function; + + // Compare address of hook target. + if (*p == func_old) + { + //printf (" Hit!! %p\n", *p); + //fflush (stdout); + + DWORD before; + // Change memory protection in order to write. + VirtualProtect(p, sizeof(p), + PAGE_EXECUTE_READWRITE, &before); + // Change hook target to new function. + WriteProcessMemory(GetCurrentProcess(), + p, &func_new, sizeof(func_new), NULL); + // Revert memory protection. + VirtualProtect(p, sizeof(p), before, &before); + + //printf (" Changed!!\n"); + //fflush (stdout); + } + pthunk++; + } + pimp_desc++; + } + } +} + +#endif // __MINGW32__ diff --git a/flower/mingw-utf8-main.cc b/flower/mingw-utf8-main.cc new file mode 100644 index 0000000..47771a1 --- /dev/null +++ b/flower/mingw-utf8-main.cc @@ -0,0 +1,84 @@ +/* + This file is part of LilyPond, the GNU music typesetter. + + Copyright (C) 2015 Masamichi Hosoda <truer...@trueroad.jp> + + LilyPond is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + LilyPond is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with LilyPond. If not, see <http://www.gnu.org/licenses/>. +*/ + +#ifdef __MINGW32__ + +#include <cwchar> +#include <vector> + +#include "mingw-utf8.hh" +#include "mingw-utf8-hook.hh" +#include "mingw-utf8-conv.hh" + +extern "C" +int __wgetmainargs (int*, wchar_t***, wchar_t***, int, int*); + +#undef main + +// Replaced main() +int main (int, char**, char**) +{ + // Hook filename related functions. + mingw_utf8_hook(); + + // Get UTF-16 commandline arguments and environment variables. + int argc, si=0; + wchar_t **wargv; + wchar_t **wenvp; + __wgetmainargs (&argc, &wargv, &wenvp, 1, &si); + + // Convert commandline arguments from UTF-16 to UTF-8. + std::vector<std::vector<char> > argv_vvc (argc); + std::vector<char*> argv_vc (argc); + + for (int i = 0; i < argc; i++) + { + argv_vvc.at(i) = mingw_utf8_16to8 (wargv[i]); + argv_vc.at(i) = argv_vvc.at(i).data(); + } + argv_vc.push_back (NULL); + + // Count environment variables. + wchar_t **wenv_tmp=wenvp; + int env_count=0; + while(*wenv_tmp) + { + wenv_tmp++; + env_count++; + } + + // Convert environment variables from UTF-16 to UTF-8. + std::vector<std::vector<char> > env_vvc (env_count); + std::vector<char*> env_vc (env_count); + + for(int i = 0; i < env_count; i++) + { + env_vvc.at(i) = mingw_utf8_16to8 (wenvp[i]); + env_vc.at(i) = env_vvc.at(i).data(); + } + env_vc.push_back (NULL); + + // Call original main(). + char **argv = argv_vc.data(); + char **envp = env_vc.data(); + + return utf8_main (argc, argv, envp); +} + +#endif // __MINGW32__ diff --git a/lily/GNUmakefile b/lily/GNUmakefile index 6d9afd7..25ab6e3 100644 --- a/lily/GNUmakefile +++ b/lily/GNUmakefile @@ -25,6 +25,7 @@ CXXFLAGS += -Woverloaded-virtual # ifeq ($(PLATFORM_WINDOWS),yes) +CONFIG_LDFLAGS += -ldbghelp WINDRES_FLAGS += -DLilyPond=0 -DLY=1 O_FILES += $(outdir)/lilypond.rc.o $(outdir)/lilypond: $(outdir)/lilypond.rc.o diff --git a/lily/main.cc b/lily/main.cc index 808274c..e552702 100644 --- a/lily/main.cc +++ b/lily/main.cc @@ -56,6 +56,10 @@ using namespace std; #include "version.hh" #include "warn.hh" +#ifdef __MINGW32__ +#include "mingw-utf8.hh" +#endif + /* * Global options that can be overridden through command line. */ -- 2.1.4
_______________________________________________ lilypond-devel mailing list lilypond-devel@gnu.org https://lists.gnu.org/mailman/listinfo/lilypond-devel