> Seems I found 2 errors in .ar command of sqlite3 on Windows:
>
First, in shell.c the code that reads lines from console as input uses
> wrong codepage code CP_ACP or CP_OEMCP.
> This doesn't allow to pass proper filenames to your func runOneSqlLine() ,
> as my Windows 10 console uses codepage 1251.
> A made changes to shell.c:utf8_printf(), local_getline() and
> os_win.c:winMbcsToUnicode(), winUnicodeToMbcs().
>
> Second, on Windows we have wide chars filenames, and your code uses utf8
> for filenames.
>
This doesn't allow the .ar commands to work with non ASCII filenames
> (charcodes > 80).
>
I made changes to fileio.c:writeFile(), struct FsdirLevel, fsdirNext().
>
> Both errors make impossible to use archive command with files with
> nonASCII names.
> My fixes have comment //za++, you may search for it. I attached the patch.
>
>
> I compile with mingw gcc for Windows 10.
> Bellow is my gcc version.
>
> At the end of the this email there is a patch against your trunk branch
> .\sqlite3.exe --version 3.29.0 2019-05-03 02:41:36
> 9b5d943426c9273162ecb4c561eb3b25e843318dd438239c882c9db50f78alt2
>
> I use this compile command from powershell:
> (please pay attention that SQLITE_WIN32_HAS_WIDE must be defined
> explicitly because here we have shell.c and sqlite3.c compilation units.
> Shell.c doesn't define SQLITE_WIN32_HAS_WIDE by itself.)
>
> gcc -municode -mwin32 -g3 -Og -DSQLITE_WIN32_HAS_WIDE
> -DSQLITE_OS_WINNT=1 -DSQLITE_DEBUG -DSQLITE_SHELL_IS
> _UTF8=0 -I. -I/f/Zavla_VB/sqlite_fossil/sqlite/src
> -I/f/Zavla_VB/sqlite_fossil/sqlite/ext/rtree -I/f/Zavla_VB/sqlite_fos
> sil/sqlite/ext/icu -I/f/Zavla_VB/sqlite_fossil/sqlite/ext/fts3
> -I/f/Zavla_VB/sqlite_fossil/sqlite/ext/async -I/f/Zavla_V
> B/sqlite_fossil/sqlite/ext/session
> -I/f/Zavla_VB/sqlite_fossil/sqlite/ext/userauth -D_HAVE_SQLITE_CONFIG_H
> -DBUILD_sqlit
> e -DNDEBUG '-I/mingw64/include/tcl8.6' -DSQLITE_THREADSAFE=1
> -DSQLITE_HAVE_ZLIB=1 -DHAVE_READLINE=0 -DHAVE_EDITLINE=0 -D
> SQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE
> -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_UNKNOWN_
> SQL_FUNCTION -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBPAGE_VTAB
> -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_OFFSET_SQL
> _FUNC -DSQLITE_ENABLE_DESERIALIZE -DSQLITE_INTROSPECTION_PRAGMAS
> -ID:\msys32\mingw32\include\ -o .libs/sqlite3.exe shel
> l.c sqlite3.c -lz -L/mingw64/lib -LD:\msys32\mingw64\lib
>
>
> My versions:
>
> PS F:\Zavla_VB\sqlite_fossil\sqlite> cd .\.libs\
> PS F:\Zavla_VB\sqlite_fossil\sqlite\.libs> .\sqlite3.exe --version
> 3.29.0 2019-05-03 02:41:36
> 9b5d943426c9273162ecb4c561eb3b25e843318dd438239c882c9db50f78alt2
>
> PS F:\Zavla_VB\sqlite_fossil\sqlite\.libs> cd ..
> PS F:\Zavla_VB\sqlite_fossil\sqlite> fossil.exe status
> repository: F:/Zavla_VB/sqlite_fossil/sqlite/..\sqlite.fossil
> local-root: F:/Zavla_VB/sqlite_fossil/sqlite/
> config-db: C:/Users/z.malinovskiy/AppData/Local/_fossil
> checkout: 9b5d943426c9273162ecb4c561eb3b25e843318d 2019-05-03 02:41:36
> UTC
> parent: 48889530a9de22fee536edfd1627be62396ed18d 2019-05-02 17:45:52
> UTC
> tags: trunk
> comment: Fix the ".open --hexdb" command in the CLI so that it works
> even with terminal input. (user: drh)
>
> PS F:\Zavla_VB\sqlite_fossil\sqlite> gcc --version
> gcc.exe (GCC) 8.2.0
> Copyright (C) 2018 Free Software Foundation, Inc.
>
> Next is a patch with my fixes:
> ============ HERE GOES THE PATCH ================
>
>
> Index: ext/misc/fileio.c
> ==================================================================
> --- ext/misc/fileio.c
> +++ ext/misc/fileio.c
> @@ -98,11 +98,48 @@
> # endif
> # ifndef stat
> # define stat _stat
> # endif
> # define mkdir(path,mode) _mkdir(path)
> -# define lstat(path,buf) stat(path,buf)
> +//za++ wide Windows expects wchar_t filenames
> +#ifdef SQLITE_WIN32_HAS_WIDE
> + #define closedir(X) _wclosedir((X))
> + #define opendir(X) opendir_onWideWindows((X))
> + _WDIR* opendir_onWideWindows(const char* utfFilename){
> + LPWSTR tempws = sqlite3_win32_utf8_to_unicode(utfFilename);
> + _WDIR* res = _wopendir(tempws);
> + sqlite3_free(tempws);
> + return res;
> + }
> +
> + #define fopen(X,Y) fopen_onWideWindows((X),(Y))
> + FILE* fopen_onWideWindows(const char* utfFilename, const char* mode) {
> +
> + LPWSTR tempws = sqlite3_win32_utf8_to_unicode(utfFilename);
> + if (tempws == 0) return 0;
> + wchar_t wmode[] = L"\0\0\0\0\0\0";
> + int len = strlen(mode);
> + int i;
> + for (i = 0; i<len; i++) {
> + wmode[i] = (wchar_t)mode[i];
> + }
> + FILE* res = _wfopen(tempws,wmode);
> + sqlite3_free(tempws);
> + return res;
> + }
> + #define lstat(X,Y) lstat_onWideWindows((X),(Y))
> + int lstat_onWideWindows(const char* utfFilename, struct stat* pStatBuf){
> + LPWSTR tempws = sqlite3_win32_utf8_to_unicode(utfFilename);
> + if (tempws == 0) return 0;
> + int res = _wstat64(tempws, pStatBuf);
> + sqlite3_free(tempws);
> + return res;
> + }
> +#else
> +# define lstat(path,buf) stat(path,buf)
> +#endif //SQLITE_WIN32_HAS_WIDE
> +
> #endif
> #include <time.h>
> #include <errno.h>
>
>
> @@ -261,11 +298,12 @@
> static int fileStat(
> const char *zPath,
> struct stat *pStatBuf
> ){
> #if defined(_WIN32)
> - int rc = stat(zPath, pStatBuf);
> + //za++
> + int rc = lstat(zPath, pStatBuf);
> if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
> return rc;
> #else
> return stat(zPath, pStatBuf);
> #endif
> @@ -352,10 +390,19 @@
> const char *zTo = (const char*)sqlite3_value_text(pData);
> if( symlink(zTo, zFile)<0 ) return 1;
> }else
> #endif
> {
> + //za++ Windows expects wchar_t filenames
> +#ifdef SQLITE_WIN32_HAS_WIDE
> + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
> + LPWSTR tempws = sqlite3_win32_utf8_to_unicode(zFile);
> + #pragma push_macro("mkdir")
> + #pragma push_macro("chmode")
> + #define mkdir(X,Y) _wmkdir((tempws))
> + #define chmod(X,Y) _wchmod((tempws),(Y))
> +#endif
> if( S_ISDIR(mode) ){
> if( mkdir(zFile, mode) ){
> /* The mkdir() call to create the directory failed. This might not
> ** be an error though - if there is already a directory at the
> same
> ** path and either the permissions already match or can be changed
> @@ -364,19 +411,31 @@
> if( errno!=EEXIST
> || 0!=fileStat(zFile, &sStat)
> || !S_ISDIR(sStat.st_mode)
> || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile,
> mode&0777))
> ){
> + //za++ Windows expects wchar_t filenames
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + sqlite3_free(tempws);
> + #else
> + #endif
> return 1;
> }
> }
> }else{
> sqlite3_int64 nWrite = 0;
> const char *z;
> int rc = 0;
> FILE *out = fopen(zFile, "wb");
> - if( out==0 ) return 1;
> + if( out==0 ) {
> + //za++ Windows expects wchar_t filenames
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + sqlite3_free(tempws);
> + #else
> + #endif
> + return 1;
> + }
> z = (const char*)sqlite3_value_blob(pData);
> if( z ){
> sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
> nWrite = sqlite3_value_bytes(pData);
> if( nWrite!=n ){
> @@ -385,13 +444,30 @@
> }
> fclose(out);
> if( rc==0 && mode && chmod(zFile, mode & 0777) ){
> rc = 1;
> }
> - if( rc ) return 2;
> + if( rc ) {
> + //za++ Windows expects wchar_t filenames
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + sqlite3_free(tempws);
> + #else
> + #endif
> + return 2;
> + }
> sqlite3_result_int64(pCtx, nWrite);
> }
> + //za++ Windows expects wchar_t filenames
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + sqlite3_free(tempws);
> + //#undef fopen
> + #undef mkdir
> + #undef chmode
> + //#pragma pop_macro("fopen")
> + #pragma pop_macro("mkdir")
> + #pragma pop_macro("chmode")
> + #endif
> }
>
> if( mtime>=0 ){
> #if defined(_WIN32)
> /* Windows */
> @@ -538,11 +614,16 @@
> */
> typedef struct fsdir_cursor fsdir_cursor;
> typedef struct FsdirLevel FsdirLevel;
>
> struct FsdirLevel {
> + //za++
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + _WDIR *pDir; /* From opendir() */
> + #else
> DIR *pDir; /* From opendir() */
> + #endif
> char *zDir; /* Name of directory (nul-terminated) */
> };
>
> struct fsdir_cursor {
> sqlite3_vtab_cursor base; /* Base class - must be first */
> @@ -683,25 +764,37 @@
>
> pLvl->zDir = pCur->zPath;
> pCur->zPath = 0;
> pLvl->pDir = opendir(pLvl->zDir);
> if( pLvl->pDir==0 ){
> - fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
> + fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
> //za++ may be pLvl->zDir
> return SQLITE_ERROR;
> }
> }
>
> while( pCur->iLvl>=0 ){
> FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
> + //za++
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + struct _wdirent *pEntry = _wreaddir(pLvl->pDir);
> + #else
> struct dirent *pEntry = readdir(pLvl->pDir);
> + #endif
> if( pEntry ){
> if( pEntry->d_name[0]=='.' ){
> if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
> if( pEntry->d_name[1]=='\0' ) continue;
> }
> sqlite3_free(pCur->zPath);
> + //za++
> + #ifdef SQLITE_WIN32_HAS_WIDE
> + char* temputf = sqlite3_win32_unicode_to_utf8(pEntry->d_name);
> + pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, temputf);
> + sqlite3_free(temputf);
> + #else
> pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
> + #endif
> if( pCur->zPath==0 ) return SQLITE_NOMEM;
> if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
> fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
> return SQLITE_ERROR;
> }
>
> Index: shell.c
> ==================================================================
> --- shell.c
> +++ shell.c
> @@ -2335,10 +2335,11 @@
> static int fileStat(
> const char *zPath,
> struct stat *pStatBuf
> ){
> #if defined(_WIN32)
> + //za++
> int rc = lstat(zPath, pStatBuf);
> if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
> return rc;
> #else
> return stat(zPath, pStatBuf);
> @@ -2816,14 +2817,12 @@
> while( pCur->iLvl>=0 ){
> FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
> //za++
> #ifdef SQLITE_WIN32_HAS_WIDE
> struct _wdirent *pEntry = _wreaddir(pLvl->pDir);
> - #define MAYBEWIDE L
> #else
> struct dirent *pEntry = readdir(pLvl->pDir);
> - #define MAYBEWIDE
> #endif
> if( pEntry ){
> if( pEntry->d_name[0]=='.' ){
> if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
> if( pEntry->d_name[1]=='\0' ) continue;
>
> Index: src/os_win.c
> ==================================================================
> --- src/os_win.c
> +++ src/os_win.c
> @@ -1728,17 +1728,22 @@
> */
> static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){
> int nByte;
> LPWSTR zMbcsText;
> int codepage = useAnsi ? CP_ACP : CP_OEMCP;
> + //za++ wrongly determined current console codepage under win10 for
> example, my current codepage is 1251
> + if (useAnsi > 3) { //default values CP_ACP=0, CP_OEMCP=3
> + codepage = useAnsi; //we've passed a codepage
> + }
> + //za--
>
> nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL,
> 0)*sizeof(WCHAR);
> if( nByte==0 ){
> return 0;
> }
> - zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) );
> + zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); //za *sizeof
> wchar seems redundant
> if( zMbcsText==0 ){
> return 0;
> }
> nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText,
> nByte);
> @@ -1757,10 +1762,15 @@
> */
> static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){
> int nByte;
> char *zText;
> int codepage = useAnsi ? CP_ACP : CP_OEMCP;
> + //za++ wrongly determined current console codepage under win10 for
> example, my current codepage is 1251
> + if (useAnsi > 3) { //default values CP_ACP=0, CP_OEMCP=3
> + codepage = useAnsi; //we've passed a codepage
> + }
> + //za--
>
> nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0);
> if( nByte == 0 ){
> return 0;
> }
>
> Index: src/shell.c.in
> ==================================================================
> --- src/shell.c.in
> +++ src/shell.c.in
> @@ -418,12 +418,16 @@
> #if defined(_WIN32) || defined(WIN32)
> void utf8_printf(FILE *out, const char *zFormat, ...){
> va_list ap;
> va_start(ap, zFormat);
> if( stdout_is_console && (out==stdout || out==stderr) ){
> + //za++
> + int consoleHasCodePage = GetConsoleCP();
> + //za--
> +
> char *z1 = sqlite3_vmprintf(zFormat, ap);
> - char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0);
> + char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, consoleHasCodePage);
> sqlite3_free(z1);
> fputs(z2, out);
> sqlite3_free(z2);
> }else{
> vfprintf(out, zFormat, ap);
> @@ -591,11 +595,14 @@
> }
> #if defined(_WIN32) || defined(WIN32)
> /* For interactive input on Windows systems, translate the
> ** multi-byte characterset characters into UTF-8. */
> if( stdin_is_interactive && in==stdin ){
> - char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
> + //za++
> + int consoleHasCodePage = GetConsoleCP();
> + //za--
> + char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine,
> consoleHasCodePage);
> if( zTrans ){
> int nTrans = strlen30(zTrans)+1;
> if( nTrans>nLine ){
> zLine = realloc(zLine, nTrans);
> if( zLine==0 ) shell_out_of_memory();
>
>
>
> =================================================================
> С Уважением,
> Захар Владимирович Малиновский,
> программист 1с, компания UBC.
>
>
>
>
_______________________________________________
sqlite-users mailing list
sqlite-users@mailinglists.sqlite.org
http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users
