Package: libid3-3.8.3 Version: 3.8.3-4.2 Severity: important Tags: patch l10n
The attached patch fixes a few issues with id3lib. It works for me, but is untested on bigendian machines. Issues fixed: 1) I tried my best to fix the unicode borkage 2) The reader/writer was doing unconditional byte swapping for unicode strings. Since iconv already does that I removed those parts. 3) The reader/writer always prepended a Byte Order Mark for unicode strings, which it only should do for type 1 (UTF16) and never do for type 2 (UTF16BE) or type 3 (UTF8), so I changed the defaults to disabled, since iconv already takes care of the BOM for UTF16. 4) If a program like id3v2 is used in a locale _not using the iso-8859-1 charset_ (e.g. en_US.UTF-8 or ja_JP) libid3 would put invalid encoding into a iso-8859-1 field. The patch changes the default to 'native' (locale) encoding and unconvertible characters will be converted to question marks, for both reading and writing. With minimal changes a program like id3v2 can be converted to support writing proper unicode tags (see http://bugs.debian.org/232307). Test files in different encodings are available at http://ranmachan.dyndns.org/~ranma/mp3test/ -- System Information: Debian Release: testing/unstable APT prefers unstable APT policy: (990, 'unstable'), (500, 'hoary') Architecture: i386 (i686) Shell: /bin/sh linked to /bin/bash Kernel: Linux 2.6.11.12-ac7-vs1.9.5-htbatm-imq Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8) Versions of packages libid3-3.8.3 depends on: ii libc6 2.3.2.ds1-22 GNU C Library: Shared libraries an ii libgcc1 1:4.0.0-9 GCC support library ii libstdc++5 1:3.3.6-6 The GNU Standard C++ Library v3 ii zlib1g 1:1.2.2-4 compression library - runtime libid3-3.8.3 recommends no packages. -- no debconf information
diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/include/id3/globals.h id3lib3.8.3-3.8.3.patched/include/id3/globals.h --- id3lib3.8.3-3.8.3/include/id3/globals.h 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/include/id3/globals.h 2005-07-08 23:36:05.000000000 +0200 @@ -141,12 +141,13 @@ ID3TE_UTF16, ID3TE_UTF16BE, ID3TE_UTF8, + ID3TE_NATIVE, ID3TE_NUMENCODINGS, ID3TE_ASCII = ID3TE_ISO8859_1, // do not use this -> use ID3TE_IS_SINGLE_BYTE_ENC(enc) instead ID3TE_UNICODE = ID3TE_UTF16 // do not use this -> use ID3TE_IS_DOUBLE_BYTE_ENC(enc) instead }; -#define ID3TE_IS_SINGLE_BYTE_ENC(enc) ((enc) == ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8) +#define ID3TE_IS_SINGLE_BYTE_ENC(enc) ((enc) == ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8 || (enc) == ID3TE_NATIVE) #define ID3TE_IS_DOUBLE_BYTE_ENC(enc) ((enc) == ID3TE_UTF16 || (enc) == ID3TE_UTF16BE) /** Enumeration of the various id3 specifications diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/include/id3/io_helpers.h id3lib3.8.3-3.8.3.patched/include/id3/io_helpers.h --- id3lib3.8.3-3.8.3/include/id3/io_helpers.h 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/include/id3/io_helpers.h 2005-07-07 21:19:49.000000000 +0200 @@ -70,8 +70,8 @@ ID3_C_EXPORT size_t writeString(ID3_Writer&, String); ID3_C_EXPORT size_t writeText(ID3_Writer&, String); - ID3_C_EXPORT size_t writeUnicodeString(ID3_Writer&, String, bool = true); - ID3_C_EXPORT size_t writeUnicodeText(ID3_Writer&, String, bool = true); + ID3_C_EXPORT size_t writeUnicodeString(ID3_Writer&, String, bool = false); + ID3_C_EXPORT size_t writeUnicodeText(ID3_Writer&, String, bool = false); ID3_C_EXPORT size_t writeBENumber(ID3_Writer&, uint32 val, size_t); ID3_C_EXPORT size_t writeTrailingSpaces(ID3_Writer&, String, size_t); ID3_C_EXPORT size_t writeUInt28(ID3_Writer&, uint32); diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field.cpp id3lib3.8.3-3.8.3.patched/src/field.cpp --- id3lib3.8.3-3.8.3/src/field.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/field.cpp 2005-07-09 00:01:13.000000000 +0200 @@ -906,7 +906,7 @@ _changed(false), _fixed_size(def._fixed_size), _num_items(0), - _enc((_type == ID3FTY_TEXTSTRING) ? ID3TE_ASCII : ID3TE_NONE) + _enc((_type == ID3FTY_TEXTSTRING) ? ID3TE_NATIVE : ID3TE_NONE) { this->Clear(); } @@ -943,11 +943,11 @@ _text.erase(); if (_fixed_size > 0) { - if (this->GetEncoding() == ID3TE_UNICODE) + if (ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) { _text.assign(_fixed_size * 2, '\0'); } - else if (this->GetEncoding() == ID3TE_ASCII) + else if (ID3TE_IS_SINGLE_BYTE_ENC(this->GetEncoding())) { _text.assign(_fixed_size, '\0'); } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field_string_ascii.cpp id3lib3.8.3-3.8.3.patched/src/field_string_ascii.cpp --- id3lib3.8.3-3.8.3/src/field_string_ascii.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/field_string_ascii.cpp 2005-07-09 00:41:13.000000000 +0200 @@ -48,6 +48,7 @@ if ((this->GetType() == ID3FTY_TEXTSTRING) && data) { String str(data); + this->SetEncoding(ID3TE_NATIVE); len = this->SetText_i(str); } return len; @@ -86,38 +87,45 @@ **/ size_t ID3_FieldImpl::Get(char* buffer, size_t maxLength) const { + ID3_TextEnc enc = this->GetEncoding(); size_t size = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII && - buffer != NULL && maxLength > 0) + if (this->GetType() != ID3FTY_TEXTSTRING || + buffer == NULL || maxLength <= 0) { - String data = this->GetText(); - size = dami::min(maxLength, data.size()); - ::memcpy(buffer, data.data(), size); - if (size < maxLength) - { - buffer[size] = '\0'; - } + return 0; } + String data = dami::convert(this->GetText(), enc, ID3TE_NATIVE); + /* + * maxLenght-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + size = dami::min(maxLength-1, data.size()); + data.copy(buffer, size); + buffer[size] = '\0'; return size; } size_t ID3_FieldImpl::Get(char* buf, size_t maxLen, size_t index) const { + ID3_TextEnc enc = this->GetEncoding(); size_t size = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII && - buf != NULL && maxLen > 0) + if (this->GetType() != ID3FTY_TEXTSTRING || + buf == NULL || maxLen <= 0) { - String data = this->GetTextItem(index); - size = dami::min(maxLen, data.size()); - ::memcpy(buf, data.data(), size); - if (size < maxLen) - { - buf[size] = '\0'; - } + return 0; } + String data = dami::convert(this->GetTextItem(index), enc, ID3TE_NATIVE); + /* + * maxLen-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + size = dami::min(maxLen-1, data.size()); + data.copy(buf, size); + buf[size] = '\0'; + return size; } @@ -135,7 +143,7 @@ { String data; if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII) + ID3TE_IS_SINGLE_BYTE_ENC(this->GetEncoding())) { const char* raw = this->GetRawTextItem(index); if (raw != NULL) @@ -225,7 +233,7 @@ // ASSERT(_fixed_size == 0) _text += '\0'; - if (this->GetEncoding() == ID3TE_UNICODE) + if (ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) { _text += '\0'; } @@ -261,8 +269,7 @@ const char* ID3_FieldImpl::GetRawText() const { const char* text = NULL; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII) + if (this->GetType() == ID3FTY_TEXTSTRING) { text = _text.c_str(); } @@ -271,16 +278,21 @@ const char* ID3_FieldImpl::GetRawTextItem(size_t index) const { + ID3_TextEnc enc = this->GetEncoding(); const char* text = NULL; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_ASCII && - index < this->GetNumTextItems()) + if (this->GetType() != ID3FTY_TEXTSTRING || + index >= this->GetNumTextItems()) { - text = _text.c_str(); - for (size_t i = 0; i < index; ++i) - { - text += strlen(text) + 1; - } + return NULL; + } + if (ID3TE_IS_DOUBLE_BYTE_ENC(enc)) + { + return (char *) this->GetRawUnicodeTextItem(index); + } + text = _text.c_str(); + for (size_t i = 0; i < index; ++i) + { + text += strlen(text) + 1; } return text; } @@ -289,7 +301,7 @@ { String readEncodedText(ID3_Reader& reader, size_t len, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::readText(reader, len); } @@ -298,7 +310,7 @@ String readEncodedString(ID3_Reader& reader, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::readString(reader); } @@ -307,20 +319,20 @@ size_t writeEncodedText(ID3_Writer& writer, String data, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::writeText(writer, data); } - return io::writeUnicodeText(writer, data); + return io::writeUnicodeText(writer, data, false); } size_t writeEncodedString(ID3_Writer& writer, String data, ID3_TextEnc enc) { - if (enc == ID3TE_ASCII) + if (ID3TE_IS_SINGLE_BYTE_ENC(enc)) { return io::writeString(writer, data); } - return io::writeUnicodeString(writer, data); + return io::writeUnicodeString(writer, data, false); } } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/field_string_unicode.cpp id3lib3.8.3-3.8.3.patched/src/field_string_unicode.cpp --- id3lib3.8.3-3.8.3/src/field_string_unicode.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/field_string_unicode.cpp 2005-07-09 00:00:08.000000000 +0200 @@ -49,10 +49,10 @@ size_t ID3_FieldImpl::Set(const unicode_t* data) { size_t size = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && data) + if (this->GetType() == ID3FTY_TEXTSTRING && data) { String text((const char*) data, ucslen(data) * 2); + this->SetEncoding(ID3TE_UTF16); size = this->SetText_i(text); } return size; @@ -62,7 +62,7 @@ { size_t size = 0; if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE) + ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding())) { String text((const char*) data, ucslen(data) * 2); size = this->AddText_i(text); @@ -93,26 +93,27 @@ size_t ID3_FieldImpl::Get(unicode_t *buffer, size_t maxLength) const { size_t length = 0; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && - buffer != NULL && maxLength > 0) + if (this->GetType() != ID3FTY_TEXTSTRING || + buffer == NULL || maxLength <= 0) { - size_t size = this->Size(); - length = dami::min(maxLength, size); - ::memcpy((void *)buffer, (void *)_text.data(), length * 2); - if (length < maxLength) - { - buffer[length] = NULL_UNICODE; - } + return 0; } + String utf = dami::convert(_text, this->GetEncoding(), ID3TE_UTF16); + /* + * maxLenght-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + length = dami::min(maxLength-1, utf.size()/2); + utf.copy((char*)buffer, length * 2); + buffer[length] = NULL_UNICODE; return length; } const unicode_t* ID3_FieldImpl::GetRawUnicodeText() const { const unicode_t* text = NULL; - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE) + if (this->GetType() == ID3FTY_TEXTSTRING) { text = (unicode_t *)_text.data(); } @@ -123,7 +124,7 @@ { const unicode_t* text = NULL; if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && + ID3TE_IS_DOUBLE_BYTE_ENC(this->GetEncoding()) && index < this->GetNumTextItems()) { String unicode = _text + '\0' + '\0'; @@ -140,23 +141,28 @@ { size_t length = 0; size_t total_items = this->GetNumTextItems(); - if (this->GetType() == ID3FTY_TEXTSTRING && - this->GetEncoding() == ID3TE_UNICODE && - buffer != NULL && maxLength > 0 && itemNum < total_items) + if (this->GetType() != ID3FTY_TEXTSTRING || + buffer == NULL || maxLength <= 0 || itemNum >= total_items) { - const unicode_t* text = this->GetRawUnicodeTextItem(itemNum); - if (NULL != text) - { - size_t length = dami::min(maxLength, ucslen(text)); - ::memcpy(buffer, text, length * 2); - if (length < maxLength) - { - buffer[length] = NULL_UNICODE; - } - } + return 0; } + const unicode_t *raw = this->GetRawUnicodeTextItem(itemNum); + if (raw == NULL) + { + return 0; + } + String s; + s.append((char*) raw, ucslen(raw)*2); + String data = dami::convert(s, this->GetEncoding(), ID3TE_UTF16); + /* + * maxLenght-1 is used to leave room for the terminating \0. + * Safe by default is good. + * http://www.courtesan.com/todd/papers/strlcpy.html + */ + length = dami::min(maxLength-1, data.size()/2); + data.copy((char*)buffer, length * 2); + buffer[length] = NULL_UNICODE; + return length; } - - diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/helpers.cpp id3lib3.8.3-3.8.3.patched/src/helpers.cpp --- id3lib3.8.3-3.8.3/src/helpers.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/helpers.cpp 2005-07-09 00:23:16.000000000 +0200 @@ -51,13 +51,19 @@ { return ""; } - ID3_TextEnc enc = fp->GetEncoding(); - fp->SetEncoding(ID3TE_ASCII); + + const char *rawtext = fp->GetRawText(); + if (rawtext == NULL) + { + return ""; + } + String raw; + raw.append(rawtext, fp->Size()); + raw.append("\0\0", 2); /* make sure we have enough terminating zeroes */ - String text(fp->GetRawText(), fp->Size()); + String res = convert(raw, fp->GetEncoding(), ID3TE_NATIVE); - fp->SetEncoding(enc); - return text; + return res; } String id3::v2::getStringAtIndex(const ID3_Frame* frame, ID3_FieldID fldName, @@ -69,16 +75,32 @@ } String text; ID3_Field* fp = frame->GetField(fldName); - if (fp && fp->GetNumTextItems() < nIndex) + if (fp && fp->GetNumTextItems() >= nIndex) + { + return ""; + } + const char *rawtext = fp->GetRawTextItem(nIndex); + if (rawtext == NULL) + { + return ""; + } + size_t rawlen = 0; + ID3_TextEnc enc = fp->GetEncoding(); + if (ID3TE_IS_DOUBLE_BYTE_ENC(enc)) + { + rawlen = ucslen((unicode_t *)rawtext)*2; + } + else { - ID3_TextEnc enc = fp->GetEncoding(); - fp->SetEncoding(ID3TE_ASCII); + rawlen = strlen(rawtext); + } + String raw; + raw.append(rawtext, rawlen); + raw.append("\0\0", 2); /* make sure we have enough terminating zeroes */ - text = fp->GetRawTextItem(nIndex); + String res = convert(raw, enc, ID3TE_NATIVE); - fp->SetEncoding(enc); - } - return text; + return res; } size_t id3::v2::removeFrames(ID3_TagImpl& tag, ID3_FrameID id) diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/io_helpers.cpp id3lib3.8.3-3.8.3.patched/src/io_helpers.cpp --- id3lib3.8.3-3.8.3/src/io_helpers.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/io_helpers.cpp 2005-07-09 00:44:21.000000000 +0200 @@ -142,16 +142,8 @@ { break; } - if (bom == -1) - { - unicode += static_cast<char>(ch2); - unicode += static_cast<char>(ch1); - } - else - { - unicode += static_cast<char>(ch1); - unicode += static_cast<char>(ch2); - } + unicode += static_cast<char>(ch1); + unicode += static_cast<char>(ch2); } return unicode; } @@ -170,24 +162,8 @@ { unicode += ch1; unicode += ch2; - unicode += readText(reader, len); - } - else if (bom == 1) - { - unicode = readText(reader, len); - } - else - { - for (size_t i = 0; i < len; i += 2) - { - if (!readTwoChars(reader, ch1, ch2)) - { - break; - } - unicode += ch2; - unicode += ch1; - } } + unicode += readText(reader, len); return unicode; } @@ -363,12 +339,8 @@ // Write the BOM: 0xFEFF unicode_t BOM = 0xFEFF; writer.writeChars((const unsigned char*) &BOM, 2); - for (size_t i = 0; i < size; i += 2) - { - unicode_t ch = (data[i] << 8) | data[i+1]; - writer.writeChars((const unsigned char*) &ch, 2); - } } + writer.writeChars(&data[0], size); return writer.getCur() - beg; } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/misc_support.cpp id3lib3.8.3-3.8.3.patched/src/misc_support.cpp --- id3lib3.8.3-3.8.3/src/misc_support.cpp 2005-07-08 00:52:31.000000000 +0200 +++ id3lib3.8.3-3.8.3.patched/src/misc_support.cpp 2005-07-09 00:15:52.000000000 +0200 @@ -31,8 +31,9 @@ #include "misc_support.h" //#include "field.h" #include "id3/utils.h" // has <config.h> "id3/id3lib_streams.h" "id3/globals.h" "id3/id3lib_strings.h" +#include "id3/helpers.h" -//using namespace dami; +using namespace dami; char *ID3_GetString(const ID3_Frame *frame, ID3_FieldID fldName) { @@ -41,13 +42,10 @@ ID3_Field* fld; if (NULL != frame && NULL != (fld = frame->GetField(fldName))) { -// ID3_Field* fld = frame->GetField(fldName); - ID3_TextEnc enc = fld->GetEncoding(); - fld->SetEncoding(ID3TE_ISO8869_1); - size_t nText = fld->Size(); + String str = id3::v2::getString(frame, fldName); + size_t nText = str.size(); text = new char[nText + 1]; - fld->Get(text, nText + 1); - fld->SetEncoding(enc); + str.copy(text, nText); } return text; } @@ -57,9 +55,10 @@ char *text = NULL; if (NULL != frame) { - size_t nText = frame->GetField(fldName)->Size(); + String str = id3::v2::getStringAtIndex(frame, fldName, nIndex); + size_t nText = str.size(); text = new char[nText + 1]; - frame->GetField(fldName)->Get(text, nText + 1, nIndex); + str.copy(text, nText); } return text; } diff -Naru -x Makefile.in -x Makefile -x libtool -x control -x config.h -x configure -x config.log -x config.status id3lib3.8.3-3.8.3/src/utils.cpp id3lib3.8.3-3.8.3.patched/src/utils.cpp --- id3lib3.8.3-3.8.3/src/utils.cpp 2003-03-02 01:23:00.000000000 +0100 +++ id3lib3.8.3-3.8.3.patched/src/utils.cpp 2005-07-08 23:48:42.000000000 +0200 @@ -41,6 +41,8 @@ // check if we have all unicodes #if (defined(ID3_ICONV_FORMAT_UTF16BE) && defined(ID3_ICONV_FORMAT_UTF16) && defined(ID3_ICONV_FORMAT_UTF8) && defined(ID3_ICONV_FORMAT_ASCII)) # include <iconv.h> +# include <locale.h> +# include <langinfo.h> # include <errno.h> #else # undef HAVE_ICONV_H @@ -54,7 +56,15 @@ dami::String unicode(size * 2, '\0'); for (size_t i = 0; i < size; ++i) { - unicode[i*2+1] = toascii(data[i]); + unsigned char c = data[i]; + if ((c & 0x80) == 0) + { + unicode[i*2+1] = c; + } + else + { + unicode[i*2+1] = '?'; + } } return unicode; } @@ -66,7 +76,15 @@ dami::String ascii(size, '\0'); for (size_t i = 0; i < size; ++i) { - ascii[i] = toascii(data[i*2+1]); + static unicode_t *c = (unicode_t *)&data[i*2]; + if ((*c & 0xff80) == 0) + { + ascii[i] = *c; + } + else + { + ascii[i] = '?'; + } } return ascii; } @@ -74,13 +92,13 @@ dami::String oldconvert(dami::String data, ID3_TextEnc sourceEnc, ID3_TextEnc targetEnc) { dami::String target; -#define ID3_IS_ASCII(enc) ((enc) == ID3TE_ASCII || (enc) == ID3TE_ISO8859_1 || (enc) == ID3TE_UTF8) -#define ID3_IS_UNICODE(enc) ((enc) == ID3TE_UNICODE || (enc) == ID3TE_UTF16 || (enc) == ID3TE_UTF16BE) - if (ID3_IS_ASCII(sourceEnc) && ID3_IS_UNICODE(targetEnc)) + if (ID3TE_IS_SINGLE_BYTE_ENC(sourceEnc) && + ID3TE_IS_DOUBLE_BYTE_ENC(targetEnc)) { target = mbstoucs(data); } - else if (ID3_IS_UNICODE(sourceEnc) && ID3_IS_ASCII(targetEnc)) + else if (ID3TE_IS_DOUBLE_BYTE_ENC(sourceEnc) && + ID3TE_IS_SINGLE_BYTE_ENC(targetEnc)) { target = ucstombs(data); } @@ -117,7 +135,7 @@ namespace { - String convert_i(iconv_t cd, String source) + String convert_i(iconv_t cd, String source, bool dbcs, bool bigendian) { String target; size_t source_size = source.size(); @@ -140,19 +158,52 @@ size_t nconv = iconv(cd, &source_str, &source_size, &target_str, &target_size); - if (nconv == (size_t) -1 && errno != EINVAL && errno != E2BIG) - { -// errno is probably EILSEQ here, which means either an invalid byte sequence or a valid but unconvertible byte sequence - return target; - } target.append(buf, ID3LIB_BUFSIZ - target_size); target_str = buf; target_size = ID3LIB_BUFSIZ; + if (nconv == (size_t)-1 && (errno == EILSEQ || errno == EINVAL)) + { + if (dbcs) + { + if (bigendian) + { + source_str[0] = '\0'; + source_str[1] = '?'; + } + else + { + source_str[0] = '?'; + source_str[1] = '\0'; + } + } + else + { + source_str[0] = '?'; + } + } + else if (nconv == (size_t)-1 && errno != E2BIG) /* unexpected error */ + { + return target; + } } while (source_size > 0); return target; } + const char* getNativeCharset(void) + { + static char *charset = NULL; + char *oldlocale = NULL; + + if (charset == NULL) { + oldlocale = setlocale(LC_CTYPE, ""); + charset = nl_langinfo(CODESET); + setlocale(LC_CTYPE, oldlocale); + } + + return charset; + } + const char* getFormat(ID3_TextEnc enc) { const char* format = NULL; @@ -173,7 +224,11 @@ case ID3TE_UTF8: format = ID3_ICONV_FORMAT_UTF8; break; - + + case ID3TE_NATIVE: + format = getNativeCharset(); + break; + default: break; } @@ -196,7 +251,9 @@ iconv_t cd = iconv_open (targetFormat, sourceFormat); if (cd != (iconv_t) -1) { - target = convert_i(cd, data); + target = convert_i(cd, data, + ID3TE_IS_DOUBLE_BYTE_ENC(sourceEnc), + sourceEnc == ID3TE_UTF16BE); if (target.size() == 0) { //try it without iconv