Your message dated Fri, 19 Jun 2026 23:21:31 +0200 with message-id <[email protected]> and subject line Re: Bug#317478: libid3-3.8.3: [PATCH] fixing some id3lib unicode problems has caused the Debian Bug report #317478, regarding libid3-3.8.3: [PATCH] fixing some id3lib unicode problems to be marked as done.
This means that you claim that the problem has been dealt with. If this is not the case it is now your responsibility to reopen the Bug report if necessary, and/or fix the problem forthwith. (NB: If you are a system administrator and have no idea what this message is talking about, this may indicate a serious mail system misconfiguration somewhere. Please contact [email protected] immediately.) -- 317478: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=317478 Debian Bug Tracking System Contact [email protected] with problems
--- Begin Message ---Package: libid3-3.8.3 Version: 3.8.3-4.2 Severity: important Tags: patch l10n The attached patch fixes a few issues with id3lib. It works for me, but is untested on bigendian machines. Issues fixed: 1) I tried my best to fix the unicode borkage 2) The reader/writer was doing unconditional byte swapping for unicode strings. Since iconv already does that I removed those parts. 3) The reader/writer always prepended a Byte Order Mark for unicode strings, which it only should do for type 1 (UTF16) and never do for type 2 (UTF16BE) or type 3 (UTF8), so I changed the defaults to disabled, since iconv already takes care of the BOM for UTF16. 4) If a program like id3v2 is used in a locale _not using the iso-8859-1 charset_ (e.g. en_US.UTF-8 or ja_JP) libid3 would put invalid encoding into a iso-8859-1 field. The patch changes the default to 'native' (locale) encoding and unconvertible characters will be converted to question marks, for both reading and writing. With minimal changes a program like id3v2 can be converted to support writing proper unicode tags (see http://bugs.debian.org/232307). Test files in different encodings are available at http://ranmachan.dyndns.org/~ranma/mp3test/ -- System Information: Debian Release: testing/unstable APT prefers unstable APT policy: (990, 'unstable'), (500, 'hoary') Architecture: i386 (i686) Shell: /bin/sh linked to /bin/bash Kernel: Linux 2.6.11.12-ac7-vs1.9.5-htbatm-imq Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8) Versions of packages libid3-3.8.3 depends on: ii libc6 2.3.2.ds1-22 GNU C Library: Shared libraries an ii libgcc1 1:4.0.0-9 GCC support library ii libstdc++5 1:3.3.6-6 The GNU Standard C++ Library v3 ii zlib1g 1:1.2.2-4 compression library - runtime libid3-3.8.3 recommends no packages. -- no debconf informationdiff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/include/id3/globals.h id3lib3.8.3-3.8.3.patched/include/id3/globals.h --- id3lib3.8.3-3.8.3/include/id3/globals.h 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/include/id3/globals.h 2005-07-08 23:36:05.000000000 +0200 @@ -141,12 +141,13 @@ ID3TE_UTF16, ID3TE_UTF16BE, ID3TE_UTF8, + ID3TE_NATIVE, ID3TE_NUMENCODINGS, ID3TE_ASCII = ID3TE_ISO8859_1, // do not use this -> use ID3TE_IS_SINGLE_BYTE_ENC(enc) instead ID3TE_UNICODE = ID3TE_UTF16 // do not use this -> use ID3TE_IS_DOUBLE_BYTE_ENC(enc) instead }; -#define ID3TE_IS_SINGLE_BYTE_ENC(enc) ((enc) == ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8) +#define ID3TE_IS_SINGLE_BYTE_ENC(enc) ((enc) == ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8 || (enc) == ID3TE_NATIVE) #define ID3TE_IS_DOUBLE_BYTE_ENC(enc) ((enc) == ID3TE_UTF16 || (enc) == ID3TE_UTF16BE) /** Enumeration of the various id3 specifications diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/include/id3/io_helpers.h id3lib3.8.3-3.8.3.patched/include/id3/io_helpers.h --- id3lib3.8.3-3.8.3/include/id3/io_helpers.h 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/include/id3/io_helpers.h 2005-07-07 21:19:49.000000000 +0200 @@ -70,8 +70,8 @@ ID3_C_EXPORT size_t writeString(ID3_Writer&, String); ID3_C_EXPORT size_t writeText(ID3_Writer&, String); - ID3_C_EXPORT size_t writeUnicodeString(ID3_Writer&, String, bool = true); - ID3_C_EXPORT size_t writeUnicodeText(ID3_Writer&, String, bool = true); + ID3_C_EXPORT size_t writeUnicodeString(ID3_Writer&, String, bool = false); + ID3_C_EXPORT size_t writeUnicodeText(ID3_Writer&, String, bool = false); ID3_C_EXPORT size_t writeBENumber(ID3_Writer&, uint32 val, size_t); ID3_C_EXPORT size_t writeTrailingSpaces(ID3_Writer&, String, size_t); ID3_C_EXPORT size_t writeUInt28(ID3_Writer&, uint32); diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field.cpp id3lib3.8.3-3.8.3.patched/src/field.cpp --- id3lib3.8.3-3.8.3/src/field.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/field.cpp 2005-07-09 00:01:13.000000000 +0200 @@ -906,7 +906,7 @@ _changed(false), _fixed_size(def._fixed_size), _num_items(0), - _enc((_type == ID3FTY_TEXTSTRING) ? ID3TE_ASCII : ID3TE_NONE) + _enc((_type == ID3FTY_TEXTSTRING) ? ID3TE_NATIVE : ID3TE_NONE) { this->Clear(); } @@ -943,11 +943,11 @@ _text.erase(); if (_fixed_size > 0) { - if (this->GetEncoding() == ID3TE_UNICODE) + if (ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) { _text.assign(_fixed_size * 2, '\0'); } - else if (this->GetEncoding() == ID3TE_ASCII) + else if (ID3TE_IS_SINGLE_BYTE_ENC(this->GetEncoding())) { _text.assign(_fixed_size, '\0'); } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field_string_ascii.cpp id3lib3.8.3-3.8.3.patched/src/field_string_ascii.cpp --- id3lib3.8.3-3.8.3/src/field_string_ascii.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/field_string_ascii.cpp 2005-07-09 00:41:13.000000000 +0200 @@ -48,6 +48,7 @@ if ((this->GetType() == ID3FTY_TEXTSTRING) && data) { String str(data); + this->SetEncoding(ID3TE_NATIVE); len = this->SetText_i(str); } return len; @@ -86,38 +87,45 @@ **/ size_t ID3_FieldImpl::Get(char* buffer, size_t maxLength) const { + ID3_TextEnc enc = this->GetEncoding(); size_t size = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII && - buffer != NULL && maxLength > 0) + if (this->GetType() != ID3FTY_TEXTSTRING || + buffer == NULL || maxLength <= 0) { - String data = this->GetText(); - size = dami::min(maxLength, data.size()); - ::memcpy(buffer, data.data(), size); - if (size < maxLength) - { - buffer[size] = '\0'; - } + return 0; } + String data = dami::convert(this->GetText(), enc, ID3TE_NATIVE); + /* + * maxLenght-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + size = dami::min(maxLength-1, data.size()); + data.copy(buffer, size); + buffer[size] = '\0'; return size; } size_t ID3_FieldImpl::Get(char* buf, size_t maxLen, size_t index) const { + ID3_TextEnc enc = this->GetEncoding(); size_t size = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII && - buf != NULL && maxLen > 0) + if (this->GetType() != ID3FTY_TEXTSTRING || + buf == NULL || maxLen <= 0) { - String data = this->GetTextItem(index); - size = dami::min(maxLen, data.size()); - ::memcpy(buf, data.data(), size); - if (size < maxLen) - { - buf[size] = '\0'; - } + return 0; } + String data = dami::convert(this->GetTextItem(index), enc, ID3TE_NATIVE); + /* + * maxLen-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + size = dami::min(maxLen-1, data.size()); + data.copy(buf, size); + buf[size] = '\0'; + return size; } @@ -135,7 +143,7 @@ { String data; if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII) + ID3TE_IS_SINGLE_BYTE_ENC(this->GetEncoding())) { const char* raw = this->GetRawTextItem(index); if (raw != NULL) @@ -225,7 +233,7 @@ // ASSERT(_fixed_size == 0) _text += '\0'; - if (this->GetEncoding() == ID3TE_UNICODE) + if (ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) { _text += '\0'; } @@ -261,8 +269,7 @@ const char* ID3_FieldImpl::GetRawText() const { const char* text = NULL; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII) + if (this->GetType() == ID3FTY_TEXTSTRING) { text = _text.c_str(); } @@ -271,16 +278,21 @@ const char* ID3_FieldImpl::GetRawTextItem(size_t index) const { + ID3_TextEnc enc = this->GetEncoding(); const char* text = NULL; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII && - index < this->GetNumTextItems()) + if (this->GetType() != ID3FTY_TEXTSTRING || + index >= this->GetNumTextItems()) { - text = _text.c_str(); - for (size_t i = 0; i < index; ++i) - { - text += strlen(text) + 1; - } + return NULL; + } + if (ID3TE_IS_DOUBLE_BYTE_ENC(enc)) + { + return (char *) this->GetRawUnicodeTextItem(index); + } + text = _text.c_str(); + for (size_t i = 0; i < index; ++i) + { + text += strlen(text) + 1; } return text; } @@ -289,7 +301,7 @@ { String readEncodedText(ID3_Reader& reader, size_t len, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::readText(reader, len); } @@ -298,7 +310,7 @@ String readEncodedString(ID3_Reader& reader, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::readString(reader); } @@ -307,20 +319,20 @@ size_t writeEncodedText(ID3_Writer& writer, String data, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::writeText(writer, data); } - return io::writeUnicodeText(writer, data); + return io::writeUnicodeText(writer, data, false); } size_t writeEncodedString(ID3_Writer& writer, String data, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::writeString(writer, data); } - return io::writeUnicodeString(writer, data); + return io::writeUnicodeString(writer, data, false); } } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field_string_unicode.cpp id3lib3.8.3-3.8.3.patched/src/field_string_unicode.cpp --- id3lib3.8.3-3.8.3/src/field_string_unicode.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/field_string_unicode.cpp 2005-07-09 00:00:08.000000000 +0200 @@ -49,10 +49,10 @@ size_t ID3_FieldImpl::Set(const unicode_t* data) { size_t size = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && data) + if (this->GetType() == ID3FTY_TEXTSTRING && data) { String text((const char*) data, ucslen(data) * 2); + this->SetEncoding(ID3TE_UTF16); size = this->SetText_i(text); } return size; @@ -62,7 +62,7 @@ { size_t size = 0; if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE) + ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) { String text((const char*) data, ucslen(data) * 2); size = this->AddText_i(text); @@ -93,26 +93,27 @@ size_t ID3_FieldImpl::Get(unicode_t *buffer, size_t maxLength) const { size_t length = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && - buffer != NULL && maxLength > 0) + if (this->GetType() != ID3FTY_TEXTSTRING || + buffer == NULL || maxLength <= 0) { - size_t size = this->Size(); - length = dami::min(maxLength, size); - ::memcpy((void *)buffer, (void *)_text.data(), length * 2); - if (length < maxLength) - { - buffer[length] = NULL_UNICODE; - } + return 0; } + String utf = dami::convert(_text, this->GetEncoding(), ID3TE_UTF16); + /* + * maxLenght-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + length = dami::min(maxLength-1, utf.size()/2); + utf.copy((char*)buffer, length * 2); + buffer[length] = NULL_UNICODE; return length; } const unicode_t* ID3_FieldImpl::GetRawUnicodeText() const { const unicode_t* text = NULL; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE) + if (this->GetType() == ID3FTY_TEXTSTRING) { text = (unicode_t *)_text.data(); } @@ -123,7 +124,7 @@ { const unicode_t* text = NULL; if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && + ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding()) && index < this->GetNumTextItems()) { String unicode = _text + '\0' + '\0'; @@ -140,23 +141,28 @@ { size_t length = 0; size_t total_items = this->GetNumTextItems(); - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && - buffer != NULL && maxLength > 0 && itemNum < total_items) + if (this->GetType() != ID3FTY_TEXTSTRING || + buffer == NULL || maxLength <= 0 || itemNum >= total_items) { - const unicode_t* text = this->GetRawUnicodeTextItem(itemNum); - if (NULL != text) - { - size_t length = dami::min(maxLength, ucslen(text)); - ::memcpy(buffer, text, length * 2); - if (length < maxLength) - { - buffer[length] = NULL_UNICODE; - } - } + return 0; } + const unicode_t *raw = this->GetRawUnicodeTextItem(itemNum); + if (raw == NULL) + { + return 0; + } + String s; + s.append((char*) raw, ucslen(raw)*2); + String data = dami::convert(s, this->GetEncoding(), ID3TE_UTF16); + /* + * maxLenght-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + length = dami::min(maxLength-1, data.size()/2); + data.copy((char*)buffer, length * 2); + buffer[length] = NULL_UNICODE; + return length; } - - diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/helpers.cpp id3lib3.8.3-3.8.3.patched/src/helpers.cpp --- id3lib3.8.3-3.8.3/src/helpers.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/helpers.cpp 2005-07-09 00:23:16.000000000 +0200 @@ -51,13 +51,19 @@ { return ""; } - ID3_TextEnc enc = fp->GetEncoding(); - fp->SetEncoding(ID3TE_ASCII); + + const char *rawtext = fp->GetRawText(); + if (rawtext == NULL) + { + return ""; + } + String raw; + raw.append(rawtext, fp->Size()); + raw.append("\0\0", 2); /* make sure we have enough terminating zeroes */ - String text(fp->GetRawText(), fp->Size()); + String res = convert(raw, fp->GetEncoding(), ID3TE_NATIVE); - fp->SetEncoding(enc); - return text; + return res; } String id3::v2::getStringAtIndex(const ID3_Frame* frame, ID3_FieldID fldName, @@ -69,16 +75,32 @@ } String text; ID3_Field* fp = frame->GetField(fldName); - if (fp && fp->GetNumTextItems() < nIndex) + if (fp && fp->GetNumTextItems() >= nIndex) + { + return ""; + } + const char *rawtext = fp->GetRawTextItem(nIndex); + if (rawtext == NULL) + { + return ""; + } + size_t rawlen = 0; + ID3_TextEnc enc = fp->GetEncoding(); + if (ID3TE_IS_DOUBLE_BYTE_ENC(enc)) + { + rawlen = ucslen((unicode_t *)rawtext)*2; + } + else { - ID3_TextEnc enc = fp->GetEncoding(); - fp->SetEncoding(ID3TE_ASCII); + rawlen = strlen(rawtext); + } + String raw; + raw.append(rawtext, rawlen); + raw.append("\0\0", 2); /* make sure we have enough terminating zeroes */ - text = fp->GetRawTextItem(nIndex); + String res = convert(raw, enc, ID3TE_NATIVE); - fp->SetEncoding(enc); - } - return text; + return res; } size_t id3::v2::removeFrames(ID3_TagImpl& tag, ID3_FrameID id) diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/io_helpers.cpp id3lib3.8.3-3.8.3.patched/src/io_helpers.cpp --- id3lib3.8.3-3.8.3/src/io_helpers.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/io_helpers.cpp 2005-07-09 00:44:21.000000000 +0200 @@ -142,16 +142,8 @@ { break; } - if (bom == -1) - { - unicode += static_cast<char>(ch2); - unicode += static_cast<char>(ch1); - } - else - { - unicode += static_cast<char>(ch1); - unicode += static_cast<char>(ch2); - } + unicode += static_cast<char>(ch1); + unicode += static_cast<char>(ch2); } return unicode; } @@ -170,24 +162,8 @@ { unicode += ch1; unicode += ch2; - unicode += readText(reader, len); - } - else if (bom == 1) - { - unicode = readText(reader, len); - } - else - { - for (size_t i = 0; i < len; i += 2) - { - if (!readTwoChars(reader, ch1, ch2)) - { - break; - } - unicode += ch2; - unicode += ch1; - } } + unicode += readText(reader, len); return unicode; } @@ -363,12 +339,8 @@ // Write the BOM: 0xFEFF unicode_t BOM = 0xFEFF; writer.writeChars((const unsigned char*) &BOM, 2); - for (size_t i = 0; i < size; i += 2) - { - unicode_t ch = (data[i] << 8) | data[i+1]; - writer.writeChars((const unsigned char*) &ch, 2); - } } + writer.writeChars(&data[0], size); return writer.getCur() - beg; } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/misc_support.cpp id3lib3.8.3-3.8.3.patched/src/misc_support.cpp --- id3lib3.8.3-3.8.3/src/misc_support.cpp 2005-07-08 00:52:31.000000000 +0200 +++ id3lib3.8.3-3.8.3.patched/src/misc_support.cpp 2005-07-09 00:15:52.000000000 +0200 @@ -31,8 +31,9 @@ #include "misc_support.h" //#include "field.h" #include "id3/utils.h" // has <config.h> "id3/id3lib_streams.h" "id3/globals.h" "id3/id3lib_strings.h" +#include "id3/helpers.h" -//using namespace dami; +using namespace dami; char *ID3_GetString(const ID3_Frame *frame, ID3_FieldID fldName) { @@ -41,13 +42,10 @@ ID3_Field* fld; if (NULL != frame && NULL != (fld = frame->GetField(fldName))) { -// ID3_Field* fld = frame->GetField(fldName); - ID3_TextEnc enc = fld->GetEncoding(); - fld->SetEncoding(ID3TE_ISO8869_1); - size_t nText = fld->Size(); + String str = id3::v2::getString(frame, fldName); + size_t nText = str.size(); text = new char[nText + 1]; - fld->Get(text, nText + 1); - fld->SetEncoding(enc); + str.copy(text, nText); } return text; } @@ -57,9 +55,10 @@ char *text = NULL; if (NULL != frame) { - size_t nText = frame->GetField(fldName)->Size(); + String str = id3::v2::getStringAtIndex(frame, fldName, nIndex); + size_t nText = str.size(); text = new char[nText + 1]; - frame->GetField(fldName)->Get(text, nText + 1, nIndex); + str.copy(text, nText); } return text; } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/utils.cpp id3lib3.8.3-3.8.3.patched/src/utils.cpp --- id3lib3.8.3-3.8.3/src/utils.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/utils.cpp 2005-07-08 23:48:42.000000000 +0200 @@ -41,6 +41,8 @@ // check if we have all unicodes #if (defined(ID3_ICONV_FORMAT_UTF16BE) && defined(ID3_ICONV_FORMAT_UTF16) && defined(ID3_ICONV_FORMAT_UTF8) && defined(ID3_ICONV_FORMAT_ASCII)) # include <iconv.h> +# include <locale.h> +# include <langinfo.h> # include <errno.h> #else # undef HAVE_ICONV_H @@ -54,7 +56,15 @@ dami::String unicode(size * 2, '\0'); for (size_t i = 0; i < size; ++i) { - unicode[i*2+1] = toascii(data[i]); + unsigned char c = data[i]; + if ((c & 0x80) == 0) + { + unicode[i*2+1] = c; + } + else + { + unicode[i*2+1] = '?'; + } } return unicode; } @@ -66,7 +76,15 @@ dami::String ascii(size, '\0'); for (size_t i = 0; i < size; ++i) { - ascii[i] = toascii(data[i*2+1]); + static unicode_t *c = (unicode_t *)&data[i*2]; + if ((*c & 0xff80) == 0) + { + ascii[i] = *c; + } + else + { + ascii[i] = '?'; + } } return ascii; } @@ -74,13 +92,13 @@ dami::String oldconvert(dami::String data, ID3_TextEnc sourceEnc, ID3_TextEnc targetEnc) { dami::String target; -#define ID3_IS_ASCII(enc) ((enc) == ID3TE_ASCII || (enc) == ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8) -#define ID3_IS_UNICODE(enc) ((enc) == ID3TE_UNICODE || (enc) == ID3TE_UTF16 || (enc) == ID3TE_UTF16BE) - if (ID3_IS_ASCII(sourceEnc) && ID3_IS_UNICODE(targetEnc)) + if (ID3TE_IS_SINGLE_BYTE_ENC(sourceEnc) && + ID3TE_IS_DOUBLE_BYTE_ENC(targetEnc)) { target = mbstoucs(data); } - else if (ID3_IS_UNICODE(sourceEnc) && ID3_IS_ASCII(targetEnc)) + else if (ID3TE_IS_DOUBLE_BYTE_ENC(sourceEnc) && + ID3TE_IS_SINGLE_BYTE_ENC(targetEnc)) { target = ucstombs(data); } @@ -117,7 +135,7 @@ namespace { - String convert_i(iconv_t cd, String source) + String convert_i(iconv_t cd, String source, bool dbcs, bool bigendian) { String target; size_t source_size = source.size(); @@ -140,19 +158,52 @@ size_t nconv = iconv(cd, &source_str, &source_size, &target_str, &target_size); - if (nconv == (size_t) -1 && errno != EINVAL && errno != E2BIG) - { -// errno is probably EILSEQ here, which means either an invalid byte sequence or a valid but unconvertible byte sequence - return target; - } target.append(buf, ID3LIB_BUFSIZ - target_size); target_str = buf; target_size = ID3LIB_BUFSIZ; + if (nconv == (size_t)-1 && (errno == EILSEQ || errno == EINVAL)) + { + if (dbcs) + { + if (bigendian) + { + source_str[0] = '\0'; + source_str[1] = '?'; + } + else + { + source_str[0] = '?'; + source_str[1] = '\0'; + } + } + else + { + source_str[0] = '?'; + } + } + else if (nconv == (size_t)-1 && errno != E2BIG) /* unexpected error */ + { + return target; + } } while (source_size > 0); return target; } + const char* getNativeCharset(void) + { + static char *charset = NULL; + char *oldlocale = NULL; + + if (charset == NULL) { + oldlocale = setlocale(LC_CTYPE, ""); + charset = nl_langinfo(CODESET); + setlocale(LC_CTYPE, oldlocale); + } + + return charset; + } + const char* getFormat(ID3_TextEnc enc) { const char* format = NULL; @@ -173,7 +224,11 @@ case ID3TE_UTF8: format = ID3_ICONV_FORMAT_UTF8; break; - + + case ID3TE_NATIVE: + format = getNativeCharset(); + break; + default: break; } @@ -196,7 +251,9 @@ iconv_t cd = iconv_open (targetFormat, sourceFormat); if (cd != (iconv_t) -1) { - target = convert_i(cd, data); + target = convert_i(cd, data, + ID3TE_IS_DOUBLE_BYTE_ENC(sourceEnc), + sourceEnc == ID3TE_UTF16BE); if (target.size() == 0) { //try it without iconv
--- End Message ---
--- Begin Message ---The attached 2005 patch was un-tagged at the time ("does not appear to fix any unicode issues"). id3lib's Unicode serialisation has since been corrected separately: conditional byte-swapping/BOM handling and UTF-16/UTF-16BE/UTF-8 encoding are fixed in the current Debian patch series, and locale conversion is handled in the id3v2 front-end rather than inside the library. The issues this report listed are addressed. I'm closing this bug report. Martin On Sat, Jul 09, 2005 at 01:10:47 +0200, Tobias Diedrich wrote: > Package: libid3-3.8.3 > Version: 3.8.3-4.2 > Severity: important > Tags: patch l10n > > > The attached patch fixes a few issues with id3lib. > It works for me, but is untested on bigendian machines. > Issues fixed: > > 1) > I tried my best to fix the unicode borkage > > 2) > The reader/writer was doing unconditional byte swapping for > unicode strings. Since iconv already does that I removed those parts. > > 3) > The reader/writer always prepended a Byte Order Mark for unicode > strings, which it only should do for type 1 (UTF16) and never do for > type 2 (UTF16BE) or type 3 (UTF8), so I changed the defaults to > disabled, since iconv already takes care of the BOM for UTF16. > > 4) > If a program like id3v2 is used in a locale _not using the iso-8859-1 > charset_ (e.g. en_US.UTF-8 or ja_JP) libid3 would put invalid encoding > into a iso-8859-1 field. The patch changes the default to 'native' > (locale) encoding and unconvertible characters will be converted to > question marks, for both reading and writing. With minimal changes > a program like id3v2 can be converted to support writing proper unicode > tags (see http://bugs.debian.org/232307). > > Test files in different encodings are available at > http://ranmachan.dyndns.org/~ranma/mp3test/ > > -- System Information: > Debian Release: testing/unstable > APT prefers unstable > APT policy: (990, 'unstable'), (500, 'hoary') > Architecture: i386 (i686) > Shell: /bin/sh linked to /bin/bash > Kernel: Linux 2.6.11.12-ac7-vs1.9.5-htbatm-imq > Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8) > > Versions of packages libid3-3.8.3 depends on: > ii libc6 2.3.2.ds1-22 GNU C Library: Shared libraries > an > ii libgcc1 1:4.0.0-9 GCC support library > ii libstdc++5 1:3.3.6-6 The GNU Standard C++ Library v3 > ii zlib1g 1:1.2.2-4 compression library - runtime > > libid3-3.8.3 recommends no packages. > > -- no debconf information > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status > id3lib3.8.3-3.8.3/include/id3/globals.h > id3lib3.8.3-3.8.3.patched/include/id3/globals.h > --- id3lib3.8.3-3.8.3/include/id3/globals.h 2003-03-02 01:23:00.000000000 > +0100 > +++ id3lib3.8.3-3.8.3.patched/include/id3/globals.h 2005-07-08 > 23:36:05.000000000 +0200 > @@ -141,12 +141,13 @@ > ID3TE_UTF16, > ID3TE_UTF16BE, > ID3TE_UTF8, > + ID3TE_NATIVE, > ID3TE_NUMENCODINGS, > ID3TE_ASCII = ID3TE_ISO8859_1, // do not use this -> use > ID3TE_IS_SINGLE_BYTE_ENC(enc) instead > ID3TE_UNICODE = ID3TE_UTF16 // do not use this -> use > ID3TE_IS_DOUBLE_BYTE_ENC(enc) instead > }; > > -#define ID3TE_IS_SINGLE_BYTE_ENC(enc) ((enc) == ID3TE_ISO8859_1 || (enc) > == ID3TE_UTF8) > +#define ID3TE_IS_SINGLE_BYTE_ENC(enc) ((enc) == ID3TE_ISO8859_1 || (enc) > == ID3TE_UTF8 || (enc) == ID3TE_NATIVE) > #define ID3TE_IS_DOUBLE_BYTE_ENC(enc) ((enc) == ID3TE_UTF16 || (enc) == > ID3TE_UTF16BE) > > /** Enumeration of the various id3 specifications > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status > id3lib3.8.3-3.8.3/include/id3/io_helpers.h > id3lib3.8.3-3.8.3.patched/include/id3/io_helpers.h > --- id3lib3.8.3-3.8.3/include/id3/io_helpers.h 2003-03-02 > 01:23:00.000000000 +0100 > +++ id3lib3.8.3-3.8.3.patched/include/id3/io_helpers.h 2005-07-07 > 21:19:49.000000000 +0200 > @@ -70,8 +70,8 @@ > > ID3_C_EXPORT size_t writeString(ID3_Writer&, String); > ID3_C_EXPORT size_t writeText(ID3_Writer&, String); > - ID3_C_EXPORT size_t writeUnicodeString(ID3_Writer&, String, bool = > true); > - ID3_C_EXPORT size_t writeUnicodeText(ID3_Writer&, String, bool = > true); > + ID3_C_EXPORT size_t writeUnicodeString(ID3_Writer&, String, bool = > false); > + ID3_C_EXPORT size_t writeUnicodeText(ID3_Writer&, String, bool = > false); > ID3_C_EXPORT size_t writeBENumber(ID3_Writer&, uint32 val, size_t); > ID3_C_EXPORT size_t writeTrailingSpaces(ID3_Writer&, String, > size_t); > ID3_C_EXPORT size_t writeUInt28(ID3_Writer&, uint32); > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field.cpp > id3lib3.8.3-3.8.3.patched/src/field.cpp > --- id3lib3.8.3-3.8.3/src/field.cpp 2003-03-02 01:23:00.000000000 +0100 > +++ id3lib3.8.3-3.8.3.patched/src/field.cpp 2005-07-09 00:01:13.000000000 > +0200 > @@ -906,7 +906,7 @@ > _changed(false), > _fixed_size(def._fixed_size), > _num_items(0), > - _enc((_type == ID3FTY_TEXTSTRING) ? ID3TE_ASCII : ID3TE_NONE) > + _enc((_type == ID3FTY_TEXTSTRING) ? ID3TE_NATIVE : ID3TE_NONE) > { > this->Clear(); > } > @@ -943,11 +943,11 @@ > _text.erase(); > if (_fixed_size > 0) > { > - if (this->GetEncoding() == ID3TE_UNICODE) > + if (ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) > { > _text.assign(_fixed_size * 2, '\0'); > } > - else if (this->GetEncoding() == ID3TE_ASCII) > + else if (ID3TE_IS_SINGLE_BYTE_ENC(this->GetEncoding())) > { > _text.assign(_fixed_size, '\0'); > } > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status > id3lib3.8.3-3.8.3/src/field_string_ascii.cpp > id3lib3.8.3-3.8.3.patched/src/field_string_ascii.cpp > --- id3lib3.8.3-3.8.3/src/field_string_ascii.cpp 2003-03-02 > 01:23:00.000000000 +0100 > +++ id3lib3.8.3-3.8.3.patched/src/field_string_ascii.cpp 2005-07-09 > 00:41:13.000000000 +0200 > @@ -48,6 +48,7 @@ > if ((this->GetType() == ID3FTY_TEXTSTRING) && data) > { > String str(data); > + this->SetEncoding(ID3TE_NATIVE); > len = this->SetText_i(str); > } > return len; > @@ -86,38 +87,45 @@ > **/ > size_t ID3_FieldImpl::Get(char* buffer, size_t maxLength) const > { > + ID3_TextEnc enc = this->GetEncoding(); > size_t size = 0; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_ASCII && > - buffer != NULL && maxLength > 0) > + if (this->GetType() != ID3FTY_TEXTSTRING || > + buffer == NULL || maxLength <= 0) > { > - String data = this->GetText(); > - size = dami::min(maxLength, data.size()); > - ::memcpy(buffer, data.data(), size); > - if (size < maxLength) > - { > - buffer[size] = '\0'; > - } > + return 0; > } > + String data = dami::convert(this->GetText(), enc, ID3TE_NATIVE); > + /* > + * maxLenght-1 is used to leave room for the terminating \0. > + * Safe by default is good. > + * http://www.courtesan.com/todd/papers/strlcpy.html > + */ > + size = dami::min(maxLength-1, data.size()); > + data.copy(buffer, size); > + buffer[size] = '\0'; > > return size; > } > > size_t ID3_FieldImpl::Get(char* buf, size_t maxLen, size_t index) const > { > + ID3_TextEnc enc = this->GetEncoding(); > size_t size = 0; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_ASCII && > - buf != NULL && maxLen > 0) > + if (this->GetType() != ID3FTY_TEXTSTRING || > + buf == NULL || maxLen <= 0) > { > - String data = this->GetTextItem(index); > - size = dami::min(maxLen, data.size()); > - ::memcpy(buf, data.data(), size); > - if (size < maxLen) > - { > - buf[size] = '\0'; > - } > + return 0; > } > + String data = dami::convert(this->GetTextItem(index), enc, ID3TE_NATIVE); > + /* > + * maxLen-1 is used to leave room for the terminating \0. > + * Safe by default is good. > + * http://www.courtesan.com/todd/papers/strlcpy.html > + */ > + size = dami::min(maxLen-1, data.size()); > + data.copy(buf, size); > + buf[size] = '\0'; > + > return size; > } > > @@ -135,7 +143,7 @@ > { > String data; > if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_ASCII) > + ID3TE_IS_SINGLE_BYTE_ENC(this->GetEncoding())) > { > const char* raw = this->GetRawTextItem(index); > if (raw != NULL) > @@ -225,7 +233,7 @@ > > // ASSERT(_fixed_size == 0) > _text += '\0'; > - if (this->GetEncoding() == ID3TE_UNICODE) > + if (ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) > { > _text += '\0'; > } > @@ -261,8 +269,7 @@ > const char* ID3_FieldImpl::GetRawText() const > { > const char* text = NULL; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_ASCII) > + if (this->GetType() == ID3FTY_TEXTSTRING) > { > text = _text.c_str(); > } > @@ -271,16 +278,21 @@ > > const char* ID3_FieldImpl::GetRawTextItem(size_t index) const > { > + ID3_TextEnc enc = this->GetEncoding(); > const char* text = NULL; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_ASCII && > - index < this->GetNumTextItems()) > + if (this->GetType() != ID3FTY_TEXTSTRING || > + index >= this->GetNumTextItems()) > { > - text = _text.c_str(); > - for (size_t i = 0; i < index; ++i) > - { > - text += strlen(text) + 1; > - } > + return NULL; > + } > + if (ID3TE_IS_DOUBLE_BYTE_ENC(enc)) > + { > + return (char *) this->GetRawUnicodeTextItem(index); > + } > + text = _text.c_str(); > + for (size_t i = 0; i < index; ++i) > + { > + text += strlen(text) + 1; > } > return text; > } > @@ -289,7 +301,7 @@ > { > String readEncodedText(ID3_Reader& reader, size_t len, ID3_TextEnc enc) > { > - if (enc == ID3TE_ASCII) > + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) > { > return io::readText(reader, len); > } > @@ -298,7 +310,7 @@ > > String readEncodedString(ID3_Reader& reader, ID3_TextEnc enc) > { > - if (enc == ID3TE_ASCII) > + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) > { > return io::readString(reader); > } > @@ -307,20 +319,20 @@ > > size_t writeEncodedText(ID3_Writer& writer, String data, ID3_TextEnc enc) > { > - if (enc == ID3TE_ASCII) > + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) > { > return io::writeText(writer, data); > } > - return io::writeUnicodeText(writer, data); > + return io::writeUnicodeText(writer, data, false); > } > > size_t writeEncodedString(ID3_Writer& writer, String data, ID3_TextEnc enc) > { > - if (enc == ID3TE_ASCII) > + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) > { > return io::writeString(writer, data); > } > - return io::writeUnicodeString(writer, data); > + return io::writeUnicodeString(writer, data, false); > } > } > > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status > id3lib3.8.3-3.8.3/src/field_string_unicode.cpp > id3lib3.8.3-3.8.3.patched/src/field_string_unicode.cpp > --- id3lib3.8.3-3.8.3/src/field_string_unicode.cpp 2003-03-02 > 01:23:00.000000000 +0100 > +++ id3lib3.8.3-3.8.3.patched/src/field_string_unicode.cpp 2005-07-09 > 00:00:08.000000000 +0200 > @@ -49,10 +49,10 @@ > size_t ID3_FieldImpl::Set(const unicode_t* data) > { > size_t size = 0; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_UNICODE && data) > + if (this->GetType() == ID3FTY_TEXTSTRING && data) > { > String text((const char*) data, ucslen(data) * 2); > + this->SetEncoding(ID3TE_UTF16); > size = this->SetText_i(text); > } > return size; > @@ -62,7 +62,7 @@ > { > size_t size = 0; > if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_UNICODE) > + ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) > { > String text((const char*) data, ucslen(data) * 2); > size = this->AddText_i(text); > @@ -93,26 +93,27 @@ > size_t ID3_FieldImpl::Get(unicode_t *buffer, size_t maxLength) const > { > size_t length = 0; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_UNICODE && > - buffer != NULL && maxLength > 0) > + if (this->GetType() != ID3FTY_TEXTSTRING || > + buffer == NULL || maxLength <= 0) > { > - size_t size = this->Size(); > - length = dami::min(maxLength, size); > - ::memcpy((void *)buffer, (void *)_text.data(), length * 2); > - if (length < maxLength) > - { > - buffer[length] = NULL_UNICODE; > - } > + return 0; > } > + String utf = dami::convert(_text, this->GetEncoding(), ID3TE_UTF16); > + /* > + * maxLenght-1 is used to leave room for the terminating \0. > + * Safe by default is good. > + * http://www.courtesan.com/todd/papers/strlcpy.html > + */ > + length = dami::min(maxLength-1, utf.size()/2); > + utf.copy((char*)buffer, length * 2); > + buffer[length] = NULL_UNICODE; > return length; > } > > const unicode_t* ID3_FieldImpl::GetRawUnicodeText() const > { > const unicode_t* text = NULL; > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_UNICODE) > + if (this->GetType() == ID3FTY_TEXTSTRING) > { > text = (unicode_t *)_text.data(); > } > @@ -123,7 +124,7 @@ > { > const unicode_t* text = NULL; > if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_UNICODE && > + ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding()) && > index < this->GetNumTextItems()) > { > String unicode = _text + '\0' + '\0'; > @@ -140,23 +141,28 @@ > { > size_t length = 0; > size_t total_items = this->GetNumTextItems(); > - if (this->GetType() == ID3FTY_TEXTSTRING && > - this->GetEncoding() == ID3TE_UNICODE && > - buffer != NULL && maxLength > 0 && itemNum < total_items) > + if (this->GetType() != ID3FTY_TEXTSTRING || > + buffer == NULL || maxLength <= 0 || itemNum >= total_items) > { > - const unicode_t* text = this->GetRawUnicodeTextItem(itemNum); > - if (NULL != text) > - { > - size_t length = dami::min(maxLength, ucslen(text)); > - ::memcpy(buffer, text, length * 2); > - if (length < maxLength) > - { > - buffer[length] = NULL_UNICODE; > - } > - } > + return 0; > } > > + const unicode_t *raw = this->GetRawUnicodeTextItem(itemNum); > + if (raw == NULL) > + { > + return 0; > + } > + String s; > + s.append((char*) raw, ucslen(raw)*2); > + String data = dami::convert(s, this->GetEncoding(), ID3TE_UTF16); > + /* > + * maxLenght-1 is used to leave room for the terminating \0. > + * Safe by default is good. > + * http://www.courtesan.com/todd/papers/strlcpy.html > + */ > + length = dami::min(maxLength-1, data.size()/2); > + data.copy((char*)buffer, length * 2); > + buffer[length] = NULL_UNICODE; > + > return length; > } > - > - > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/helpers.cpp > id3lib3.8.3-3.8.3.patched/src/helpers.cpp > --- id3lib3.8.3-3.8.3/src/helpers.cpp 2003-03-02 01:23:00.000000000 +0100 > +++ id3lib3.8.3-3.8.3.patched/src/helpers.cpp 2005-07-09 00:23:16.000000000 > +0200 > @@ -51,13 +51,19 @@ > { > return ""; > } > - ID3_TextEnc enc = fp->GetEncoding(); > - fp->SetEncoding(ID3TE_ASCII); > + > + const char *rawtext = fp->GetRawText(); > + if (rawtext == NULL) > + { > + return ""; > + } > + String raw; > + raw.append(rawtext, fp->Size()); > + raw.append("\0\0", 2); /* make sure we have enough terminating zeroes */ > > - String text(fp->GetRawText(), fp->Size()); > + String res = convert(raw, fp->GetEncoding(), ID3TE_NATIVE); > > - fp->SetEncoding(enc); > - return text; > + return res; > } > > String id3::v2::getStringAtIndex(const ID3_Frame* frame, ID3_FieldID fldName, > @@ -69,16 +75,32 @@ > } > String text; > ID3_Field* fp = frame->GetField(fldName); > - if (fp && fp->GetNumTextItems() < nIndex) > + if (fp && fp->GetNumTextItems() >= nIndex) > + { > + return ""; > + } > + const char *rawtext = fp->GetRawTextItem(nIndex); > + if (rawtext == NULL) > + { > + return ""; > + } > + size_t rawlen = 0; > + ID3_TextEnc enc = fp->GetEncoding(); > + if (ID3TE_IS_DOUBLE_BYTE_ENC(enc)) > + { > + rawlen = ucslen((unicode_t *)rawtext)*2; > + } > + else > { > - ID3_TextEnc enc = fp->GetEncoding(); > - fp->SetEncoding(ID3TE_ASCII); > + rawlen = strlen(rawtext); > + } > + String raw; > + raw.append(rawtext, rawlen); > + raw.append("\0\0", 2); /* make sure we have enough terminating zeroes */ > > - text = fp->GetRawTextItem(nIndex); > + String res = convert(raw, enc, ID3TE_NATIVE); > > - fp->SetEncoding(enc); > - } > - return text; > + return res; > } > > size_t id3::v2::removeFrames(ID3_TagImpl& tag, ID3_FrameID id) > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/io_helpers.cpp > id3lib3.8.3-3.8.3.patched/src/io_helpers.cpp > --- id3lib3.8.3-3.8.3/src/io_helpers.cpp 2003-03-02 01:23:00.000000000 > +0100 > +++ id3lib3.8.3-3.8.3.patched/src/io_helpers.cpp 2005-07-09 > 00:44:21.000000000 +0200 > @@ -142,16 +142,8 @@ > { > break; > } > - if (bom == -1) > - { > - unicode += static_cast<char>(ch2); > - unicode += static_cast<char>(ch1); > - } > - else > - { > - unicode += static_cast<char>(ch1); > - unicode += static_cast<char>(ch2); > - } > + unicode += static_cast<char>(ch1); > + unicode += static_cast<char>(ch2); > } > return unicode; > } > @@ -170,24 +162,8 @@ > { > unicode += ch1; > unicode += ch2; > - unicode += readText(reader, len); > - } > - else if (bom == 1) > - { > - unicode = readText(reader, len); > - } > - else > - { > - for (size_t i = 0; i < len; i += 2) > - { > - if (!readTwoChars(reader, ch1, ch2)) > - { > - break; > - } > - unicode += ch2; > - unicode += ch1; > - } > } > + unicode += readText(reader, len); > return unicode; > } > > @@ -363,12 +339,8 @@ > // Write the BOM: 0xFEFF > unicode_t BOM = 0xFEFF; > writer.writeChars((const unsigned char*) &BOM, 2); > - for (size_t i = 0; i < size; i += 2) > - { > - unicode_t ch = (data[i] << 8) | data[i+1]; > - writer.writeChars((const unsigned char*) &ch, 2); > - } > } > + writer.writeChars(&data[0], size); > return writer.getCur() - beg; > } > > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status > id3lib3.8.3-3.8.3/src/misc_support.cpp > id3lib3.8.3-3.8.3.patched/src/misc_support.cpp > --- id3lib3.8.3-3.8.3/src/misc_support.cpp 2005-07-08 00:52:31.000000000 > +0200 > +++ id3lib3.8.3-3.8.3.patched/src/misc_support.cpp 2005-07-09 > 00:15:52.000000000 +0200 > @@ -31,8 +31,9 @@ > #include "misc_support.h" > //#include "field.h" > #include "id3/utils.h" // has <config.h> "id3/id3lib_streams.h" > "id3/globals.h" "id3/id3lib_strings.h" > +#include "id3/helpers.h" > > -//using namespace dami; > +using namespace dami; > > char *ID3_GetString(const ID3_Frame *frame, ID3_FieldID fldName) > { > @@ -41,13 +42,10 @@ > ID3_Field* fld; > if (NULL != frame && NULL != (fld = frame->GetField(fldName))) > { > -// ID3_Field* fld = frame->GetField(fldName); > - ID3_TextEnc enc = fld->GetEncoding(); > - fld->SetEncoding(ID3TE_ISO8869_1); > - size_t nText = fld->Size(); > + String str = id3::v2::getString(frame, fldName); > + size_t nText = str.size(); > text = new char[nText + 1]; > - fld->Get(text, nText + 1); > - fld->SetEncoding(enc); > + str.copy(text, nText); > } > return text; > } > @@ -57,9 +55,10 @@ > char *text = NULL; > if (NULL != frame) > { > - size_t nText = frame->GetField(fldName)->Size(); > + String str = id3::v2::getStringAtIndex(frame, fldName, nIndex); > + size_t nText = str.size(); > text = new char[nText + 1]; > - frame->GetField(fldName)->Get(text, nText + 1, nIndex); > + str.copy(text, nText); > } > return text; > } > diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x > configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/utils.cpp > id3lib3.8.3-3.8.3.patched/src/utils.cpp > --- id3lib3.8.3-3.8.3/src/utils.cpp 2003-03-02 01:23:00.000000000 +0100 > +++ id3lib3.8.3-3.8.3.patched/src/utils.cpp 2005-07-08 23:48:42.000000000 > +0200 > @@ -41,6 +41,8 @@ > // check if we have all unicodes > #if (defined(ID3_ICONV_FORMAT_UTF16BE) && defined(ID3_ICONV_FORMAT_UTF16) && > defined(ID3_ICONV_FORMAT_UTF8) && defined(ID3_ICONV_FORMAT_ASCII)) > # include <iconv.h> > +# include <locale.h> > +# include <langinfo.h> > # include <errno.h> > #else > # undef HAVE_ICONV_H > @@ -54,7 +56,15 @@ > dami::String unicode(size * 2, '\0'); > for (size_t i = 0; i < size; ++i) > { > - unicode[i*2+1] = toascii(data[i]); > + unsigned char c = data[i]; > + if ((c & 0x80) == 0) > + { > + unicode[i*2+1] = c; > + } > + else > + { > + unicode[i*2+1] = '?'; > + } > } > return unicode; > } > @@ -66,7 +76,15 @@ > dami::String ascii(size, '\0'); > for (size_t i = 0; i < size; ++i) > { > - ascii[i] = toascii(data[i*2+1]); > + static unicode_t *c = (unicode_t *)&data[i*2]; > + if ((*c & 0xff80) == 0) > + { > + ascii[i] = *c; > + } > + else > + { > + ascii[i] = '?'; > + } > } > return ascii; > } > @@ -74,13 +92,13 @@ > dami::String oldconvert(dami::String data, ID3_TextEnc sourceEnc, > ID3_TextEnc targetEnc) > { > dami::String target; > -#define ID3_IS_ASCII(enc) ((enc) == ID3TE_ASCII || (enc) == > ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8) > -#define ID3_IS_UNICODE(enc) ((enc) == ID3TE_UNICODE || (enc) == > ID3TE_UTF16 || (enc) == ID3TE_UTF16BE) > - if (ID3_IS_ASCII(sourceEnc) && ID3_IS_UNICODE(targetEnc)) > + if (ID3TE_IS_SINGLE_BYTE_ENC(sourceEnc) && > + ID3TE_IS_DOUBLE_BYTE_ENC(targetEnc)) > { > target = mbstoucs(data); > } > - else if (ID3_IS_UNICODE(sourceEnc) && ID3_IS_ASCII(targetEnc)) > + else if (ID3TE_IS_DOUBLE_BYTE_ENC(sourceEnc) && > + ID3TE_IS_SINGLE_BYTE_ENC(targetEnc)) > { > target = ucstombs(data); > } > @@ -117,7 +135,7 @@ > > namespace > { > - String convert_i(iconv_t cd, String source) > + String convert_i(iconv_t cd, String source, bool dbcs, bool bigendian) > { > String target; > size_t source_size = source.size(); > @@ -140,19 +158,52 @@ > size_t nconv = iconv(cd, > &source_str, &source_size, > &target_str, &target_size); > - if (nconv == (size_t) -1 && errno != EINVAL && errno != E2BIG) > - { > -// errno is probably EILSEQ here, which means either an invalid byte > sequence or a valid but unconvertible byte sequence > - return target; > - } > target.append(buf, ID3LIB_BUFSIZ - target_size); > target_str = buf; > target_size = ID3LIB_BUFSIZ; > + if (nconv == (size_t)-1 && (errno == EILSEQ || errno == EINVAL)) > + { > + if (dbcs) > + { > + if (bigendian) > + { > + source_str[0] = '\0'; > + source_str[1] = '?'; > + } > + else > + { > + source_str[0] = '?'; > + source_str[1] = '\0'; > + } > + } > + else > + { > + source_str[0] = '?'; > + } > + } > + else if (nconv == (size_t)-1 && errno != E2BIG) /* unexpected error */ > + { > + return target; > + } > } > while (source_size > 0); > return target; > } > > + const char* getNativeCharset(void) > + { > + static char *charset = NULL; > + char *oldlocale = NULL; > + > + if (charset == NULL) { > + oldlocale = setlocale(LC_CTYPE, ""); > + charset = nl_langinfo(CODESET); > + setlocale(LC_CTYPE, oldlocale); > + } > + > + return charset; > + } > + > const char* getFormat(ID3_TextEnc enc) > { > const char* format = NULL; > @@ -173,7 +224,11 @@ > case ID3TE_UTF8: > format = ID3_ICONV_FORMAT_UTF8; > break; > - > + > + case ID3TE_NATIVE: > + format = getNativeCharset(); > + break; > + > default: > break; > } > @@ -196,7 +251,9 @@ > iconv_t cd = iconv_open (targetFormat, sourceFormat); > if (cd != (iconv_t) -1) > { > - target = convert_i(cd, data); > + target = convert_i(cd, data, > + ID3TE_IS_DOUBLE_BYTE_ENC(sourceEnc), > + sourceEnc == ID3TE_UTF16BE); > if (target.size() == 0) > { > //try it without iconv
--- End Message ---
