This is an automated email from the ASF dual-hosted git repository.
swebb2066 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/logging-log4cxx.git
The following commit(s) were added to refs/heads/master by this push:
new acaf3886 Encode an invalid codepoint using a charset-specific
character (#730)
acaf3886 is described below
commit acaf3886293423198b5ef0da3e1c1244d7e7d805
Author: Stephen Webb <[email protected]>
AuthorDate: Wed Aug 19 09:29:54 2026 +1000
Encode an invalid codepoint using a charset-specific character (#730)
* Centralises decoding logic and uses the Unicode replacement character
codepoint to indicate a failure.
* UTF-8 encoding is available (using CharsetEncoder::getUTF8Encoder()) when
Log4cxx is built using LOG4CXX_CHARSET=utf-8. (Peviously, when Log4cxx was
built using LOG4CXX_CHARSET=utf-8, the CharsetEncoder::getUTF8Encoder()
function returned a TrivialCharsetEncoder.)
* The Telnet appender now uses the Unicode replacement character codepoint
to indicate a message encoding failure. Old behaviour is obtained by explicitly
configuring "Encoding" to "US-ASCII".
* Document the 'CharsetEncoder::encode' method behaviour and preconditions
* Remove CharsetEncoder::flush in the next ABI version
---
.github/workflows/abi-compatibility.yml | 2 -
.github/workflows/log4cxx-cpp11.yml | 3 +-
.github/workflows/log4cxx-macos.yml | 2 +-
.github/workflows/log4cxx-msys2.yml | 2 +-
.github/workflows/log4cxx-ubuntu.yml | 32 +--
.github/workflows/package_code.yml | 2 +-
.github/workflows/sonarcloud.yml | 1 -
src/fuzzers/cpp/TranscoderFuzzer.cpp | 2 +-
src/main/cpp/charsetencoder.cpp | 256 +++++++++++++++++-----
src/main/cpp/jsonlayout.cpp | 23 +-
src/main/cpp/outputstreamwriter.cpp | 1 -
src/main/cpp/telnetappender.cpp | 19 +-
src/main/cpp/transcoder.cpp | 135 ++++++------
src/main/cpp/transform.cpp | 46 +---
src/main/include/log4cxx/helpers/charsetencoder.h | 35 +--
src/main/include/log4cxx/helpers/transcoder.h | 51 ++++-
src/test/cpp/helpers/charsetencodertestcase.cpp | 7 +-
src/test/cpp/helpers/transcodertestcase.cpp | 59 ++---
18 files changed, 391 insertions(+), 287 deletions(-)
diff --git a/.github/workflows/abi-compatibility.yml
b/.github/workflows/abi-compatibility.yml
index fd5d1c06..894e890c 100644
--- a/.github/workflows/abi-compatibility.yml
+++ b/.github/workflows/abi-compatibility.yml
@@ -27,7 +27,6 @@ jobs:
job:
name: abi-check
runs-on: ubuntu-latest
- timeout-minutes: 38
strategy:
fail-fast: false
@@ -39,7 +38,6 @@ jobs:
- name: 'Configure Dependencies - Ubuntu'
run: |
- sudo apt-get update
sudo apt-get install -y g++-11 libapr1-dev libaprutil1-dev elfutils
vtable-dumper universal-ctags
- name: 'run CMake'
diff --git a/.github/workflows/log4cxx-cpp11.yml
b/.github/workflows/log4cxx-cpp11.yml
index e2f3412b..16d8928f 100644
--- a/.github/workflows/log4cxx-cpp11.yml
+++ b/.github/workflows/log4cxx-cpp11.yml
@@ -20,7 +20,6 @@ jobs:
job:
name: log4cxx-cpp11
runs-on: ubuntu-latest
- timeout-minutes: 38
steps:
- uses: actions/checkout@v6
with:
@@ -29,7 +28,6 @@ jobs:
- name: 'Configure Dependencies - Ubuntu'
run: |
- sudo apt-get update
sudo apt-get install -y libapr1-dev libaprutil1-dev libesmtp-dev
- name: 'run cmake - *nix'
@@ -42,6 +40,7 @@ jobs:
- name: run unit tests
shell: pwsh
+ timeout-minutes: 10
run: |
cd main
cd build
diff --git a/.github/workflows/log4cxx-macos.yml
b/.github/workflows/log4cxx-macos.yml
index b7ad1812..b46cd04c 100644
--- a/.github/workflows/log4cxx-macos.yml
+++ b/.github/workflows/log4cxx-macos.yml
@@ -20,7 +20,6 @@ jobs:
job:
name: ${{ matrix.os }}-${{ matrix.cxx }}-build-and-test
runs-on: ${{ matrix.os }}
- timeout-minutes: 38
strategy:
fail-fast: false
matrix:
@@ -98,6 +97,7 @@ jobs:
cmake --build .
- name: run unit tests
+ timeout-minutes: 10
run: |
cd main
cd build
diff --git a/.github/workflows/log4cxx-msys2.yml
b/.github/workflows/log4cxx-msys2.yml
index a75ed766..c859deca 100644
--- a/.github/workflows/log4cxx-msys2.yml
+++ b/.github/workflows/log4cxx-msys2.yml
@@ -20,7 +20,6 @@ jobs:
job:
name: ${{ matrix.env }}-${{ matrix.cxx }}-build-and-test
runs-on: windows-2022
- timeout-minutes: 30
strategy:
fail-fast: false
matrix:
@@ -74,6 +73,7 @@ jobs:
- name: 'run unit tests'
shell: msys2 {0}
+ timeout-minutes: 10
run: |
cd log4cxx/build
ctest -C Debug --output-on-failure
diff --git a/.github/workflows/log4cxx-ubuntu.yml
b/.github/workflows/log4cxx-ubuntu.yml
index 11367e19..91234878 100644
--- a/.github/workflows/log4cxx-ubuntu.yml
+++ b/.github/workflows/log4cxx-ubuntu.yml
@@ -20,14 +20,13 @@ jobs:
job:
name: ${{ matrix.os }}-${{ matrix.cxx }}-build-and-test
runs-on: ${{ matrix.os }}
- timeout-minutes: 38
strategy:
fail-fast: false
matrix:
- name: [ubuntu22-gcc, ubuntu22-clang, ubuntu24-gcc, ubuntu24-clang]
+ name: [ubuntu24-gcc, ubuntu24-clang, ubuntu26-gcc, ubuntu26-clang]
include:
- - name: ubuntu22-gcc
- os: ubuntu-22.04
+ - name: ubuntu24-gcc
+ os: ubuntu24.04
cxx: g++
fmt: OFF
qt: ON
@@ -39,8 +38,8 @@ jobs:
fuzzers: OFF
logchar: utf-8
next_abi: OFF
- - name: ubuntu22-clang
- os: ubuntu-22.04
+ - name: ubuntu24-clang
+ os: ubuntu24.04
cxx: clang++
fmt: OFF
qt: OFF
@@ -52,8 +51,8 @@ jobs:
fuzzers: ON
logchar: utf-8
next_abi: OFF
- - name: ubuntu24-gcc
- os: ubuntu-24.04
+ - name: ubuntu26-gcc
+ os: ubuntu-26.04
cxx: g++
fmt: ON
qt: ON
@@ -65,10 +64,10 @@ jobs:
fuzzers: OFF
logchar: utf-8
next_abi: ON
- - name: ubuntu24-clang
- os: ubuntu-24.04
+ - name: ubuntu26-clang
+ os: ubuntu-26.04
cxx: clang++
- fmt: ON
+ fmt: OFF
qt: OFF
qt6: OFF
odbc: OFF
@@ -87,22 +86,12 @@ jobs:
- name: 'Configure Dependencies'
run: |
- sudo apt-get update
sudo apt-get install -y libapr1-dev libaprutil1-dev
if [ ${{ matrix.fmt }} == ON ]; then sudo apt-get install -y
libfmt-dev; fi
if [ ${{ matrix.odbc }} == ON ]; then sudo apt-get install -y
unixodbc-dev; fi
if [ ${{ matrix.qt }} == ON ] && [ ${{ matrix.qt6 }} == OFF ]; then
sudo apt-get install -y qtbase5-dev; fi
if [ ${{ matrix.qt }} == ON ] && [ ${{ matrix.qt6 }} == ON ]; then
sudo apt-get install -y qt6-base-dev; fi
- - name: WORKAROUND FOR https://github.com/actions/runner-images/issues/8659
- if: |
- matrix.name=='ubuntu22-clang'
- run: |
- echo -e "TEMPORARY WORKAROUND FOR GITHUB RUNNER BUG #8659\n\nRemoving
GCC 13 as it breaks Clang14"
- sudo rm -f
/etc/apt/sources.list.d/ubuntu-toolchain-r-ubuntu-test-jammy.list
- sudo apt-get update
- sudo apt-get install -y --allow-downgrades libc6=2.35-*
libc6-dev=2.35-* libstdc++6=12.3.0-* libgcc-s1=12.3.0-*
-
- name: 'run cmake - posix'
run: |
ROOT=`pwd`
@@ -141,6 +130,7 @@ jobs:
fi
- name: run unit tests
+ timeout-minutes: 10
run: |
ROOT=`pwd`
cd $ROOT/build
diff --git a/.github/workflows/package_code.yml
b/.github/workflows/package_code.yml
index f512c29f..7857c74b 100644
--- a/.github/workflows/package_code.yml
+++ b/.github/workflows/package_code.yml
@@ -80,10 +80,10 @@ jobs:
- name: 'Configure Dependencies'
run: |
- sudo apt-get update
sudo apt-get install -y libapr1-dev libaprutil1-dev
- name: 'Test the archive'
+ timeout-minutes: 10
run: |
VERSION=`ls apache-log4cxx-*.tar.gz | sed -Ee
's/.*apache-log4cxx-([0-9]*)\.([0-9]*)\.([0-9]*).*/\1.\2.\3/'`
echo "Extracting files from apache-log4cxx-$VERSION.tar.gz..."
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
index 1fbe2aa4..3cd2b8dc 100644
--- a/.github/workflows/sonarcloud.yml
+++ b/.github/workflows/sonarcloud.yml
@@ -51,7 +51,6 @@ jobs:
- name: 'Configure Dependencies - Ubuntu'
run: |
- sudo apt-get update
sudo apt-get install -y libapr1-dev libaprutil1-dev
- name: Install Build Wrapper
diff --git a/src/fuzzers/cpp/TranscoderFuzzer.cpp
b/src/fuzzers/cpp/TranscoderFuzzer.cpp
index 12850a37..6ccf0188 100644
--- a/src/fuzzers/cpp/TranscoderFuzzer.cpp
+++ b/src/fuzzers/cpp/TranscoderFuzzer.cpp
@@ -136,7 +136,7 @@ namespace
if (CharsetDecoder::isError(stat))
{
- out.append(1, (logchar) Transcoder::LOSSCHAR);
+ Transcoder::encode(Transcoder::LOSSCHAR, out);
buf.increment_position(1);
}
else if (buf.position() == before)
diff --git a/src/main/cpp/charsetencoder.cpp b/src/main/cpp/charsetencoder.cpp
index ce93dc9d..eb789a2d 100644
--- a/src/main/cpp/charsetencoder.cpp
+++ b/src/main/cpp/charsetencoder.cpp
@@ -35,6 +35,12 @@
#include <stdlib.h>
#endif
+#if 15 < LOG4CXX_ABI_VERSION
+#define LOG4CXX_16_VIRTUAL_SPECIFIER override
+#else
+#define LOG4CXX_16_VIRTUAL_SPECIFIER
+#endif
+
using namespace LOG4CXX_NS;
using namespace LOG4CXX_NS::helpers;
@@ -82,7 +88,7 @@ class APRCharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
apr_status_t stat;
size_t outbytes_left = out.remaining();
@@ -116,6 +122,19 @@ class APRCharsetEncoder : public CharsetEncoder
return stat;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (codePoint <= 0x10FFFF)
+ Transcoder::encodeUTF8(codePoint, out);
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
APRCharsetEncoder(const APRCharsetEncoder&);
APRCharsetEncoder& operator=(const APRCharsetEncoder&);
@@ -141,7 +160,7 @@ class WcstombsCharsetEncoder : public CharsetEncoder
*/
log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
log4cxx_status_t stat = APR_SUCCESS;
@@ -203,7 +222,25 @@ class WcstombsCharsetEncoder : public CharsetEncoder
return stat;
}
-
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (codePoint <= 0x10FFFF && MB_LEN_MAX <=
out.remaining())
+ {
+ auto ch = static_cast<wchar_t>(codePoint);
+ auto converted = wcstombs(out.current(), &ch,
1);
+ if (static_cast<std::size_t>(-1) == converted)
+ result = APR_BADARG;
+ else
+ out.increment_position(converted);
+ }
+ else
+ result = APR_BADARG;
+ return result;
+ }
private:
WcstombsCharsetEncoder(const WcstombsCharsetEncoder&);
@@ -224,7 +261,7 @@ class USASCIICharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
log4cxx_status_t stat = APR_SUCCESS;
@@ -251,6 +288,23 @@ class USASCIICharsetEncoder : public CharsetEncoder
return stat;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (out.remaining() < 1)
+ result = APR_BADARG;
+ else if (codePoint <= 0x7F)
+ out.put(static_cast<char>(codePoint));
+ else if (Transcoder::LOSSCHAR == codePoint)
+ out.put('?');
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
USASCIICharsetEncoder(const USASCIICharsetEncoder&);
USASCIICharsetEncoder& operator=(const USASCIICharsetEncoder&);
@@ -268,33 +322,43 @@ class ISOLatinCharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
log4cxx_status_t stat = APR_SUCCESS;
- if (iter != in.end())
+ while (out.remaining() > 0 && iter != in.end())
{
- while (out.remaining() > 0 && iter != in.end())
+ LogString::const_iterator prev(iter);
+ unsigned int sv = Transcoder::decode(in, iter);
+ if (sv <= 0xFF)
+ out.put(static_cast<char>(sv));
+ else
{
- LogString::const_iterator prev(iter);
- unsigned int sv =
Transcoder::decode(in, iter);
-
- if (sv <= 0xFF)
- {
- out.put((char) sv);
- }
- else
- {
- iter = prev;
- stat = APR_BADARG;
- break;
- }
+ iter = prev;
+ stat = APR_BADARG;
+ break;
}
}
-
return stat;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (out.remaining() < 1)
+ result = APR_BADARG;
+ else if (codePoint <= 0xFF)
+ out.put(static_cast<char>(codePoint));
+ else if (Transcoder::LOSSCHAR == codePoint)
+ out.put('?');
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
ISOLatinCharsetEncoder(const ISOLatinCharsetEncoder&);
ISOLatinCharsetEncoder& operator=(const
ISOLatinCharsetEncoder&);
@@ -313,7 +377,7 @@ class TrivialCharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
if (iter != in.end())
{
@@ -334,14 +398,28 @@ class TrivialCharsetEncoder : public CharsetEncoder
return APR_SUCCESS;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (out.remaining() < 1)
+ result = APR_BADARG;
+ else if (codePoint <= 0xFF)
+ out.put(static_cast<char>(codePoint));
+ else if (Transcoder::LOSSCHAR == codePoint)
+ out.put('?');
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
TrivialCharsetEncoder(const TrivialCharsetEncoder&);
TrivialCharsetEncoder& operator=(const TrivialCharsetEncoder&);
};
-#if LOG4CXX_LOGCHAR_IS_UTF8
-typedef TrivialCharsetEncoder UTF8CharsetEncoder;
-#else
/**
* Converts a LogString to UTF-8.
*/
@@ -354,28 +432,34 @@ class UTF8CharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
while (iter != in.end() && out.remaining() >= 8)
{
- unsigned int sv = Transcoder::decode(in, iter);
-
- if (sv == 0xFFFF)
- {
- return APR_BADARG;
- }
-
+ auto sv = Transcoder::getCodePoint(in, iter);
Transcoder::encodeUTF8(sv, out);
}
return APR_SUCCESS;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (codePoint <= 0x10FFFF)
+ Transcoder::encodeUTF8(codePoint, out);
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
UTF8CharsetEncoder(const UTF8CharsetEncoder&);
UTF8CharsetEncoder& operator=(const UTF8CharsetEncoder&);
};
-#endif
/**
* Encodes a LogString to UTF16-BE.
@@ -389,23 +473,30 @@ class UTF16BECharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
while (iter != in.end() && out.remaining() >= 4)
{
- unsigned int sv = Transcoder::decode(in, iter);
-
- if (sv == 0xFFFF)
- {
- return APR_BADARG;
- }
-
+ auto sv = Transcoder::getCodePoint(in, iter);
Transcoder::encodeUTF16BE(sv, out);
}
return APR_SUCCESS;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (codePoint <= 0x10FFFF)
+ Transcoder::encodeUTF16BE(codePoint, out);
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
UTF16BECharsetEncoder(const UTF16BECharsetEncoder&);
UTF16BECharsetEncoder& operator=(const UTF16BECharsetEncoder&);
@@ -424,22 +515,29 @@ class UTF16LECharsetEncoder : public CharsetEncoder
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
- ByteBuffer& out)
+ ByteBuffer& out) override
{
while (iter != in.end() && out.remaining() >= 4)
{
- unsigned int sv = Transcoder::decode(in, iter);
-
- if (sv == 0xFFFF)
- {
- return APR_BADARG;
- }
-
+ auto sv = Transcoder::getCodePoint(in, iter);
Transcoder::encodeUTF16LE(sv, out);
}
return APR_SUCCESS;
}
+
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (codePoint <= 0x10FFFF)
+ Transcoder::encodeUTF16LE(codePoint, out);
+ else
+ result = APR_BADARG;
+ return result;
+ }
private:
UTF16LECharsetEncoder(const UTF16LECharsetEncoder&);
UTF16LECharsetEncoder& operator=(const UTF16LECharsetEncoder&);
@@ -479,10 +577,7 @@ class LocaleCharsetEncoder : public CharsetEncoder
// Encode characters that may require multiple bytes
while (nextCodePoint != in.end() && byteCount <
availableByteCount && MB_CUR_MAX <= (availableByteCount - byteCount))
{
- LogString::const_iterator lastCodePoint =
nextCodePoint;
- auto ch = Transcoder::decode(in, nextCodePoint);
- if (nextCodePoint == lastCodePoint) // invalid
input sequence?
- nextCodePoint = in.end();
+ auto ch = Transcoder::getCodePoint(in,
nextCodePoint);
auto n = std::wcrtomb(current, ch,
&this->state);
if (static_cast<std::size_t>(-1) == n) // not a
valid wide character?
{
@@ -496,6 +591,25 @@ class LocaleCharsetEncoder : public CharsetEncoder
return result;
}
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ */
+ log4cxx_status_t encode(unsigned int codePoint, ByteBuffer&
out) LOG4CXX_16_VIRTUAL_SPECIFIER
+ {
+ apr_status_t result = APR_SUCCESS;
+ if (MB_CUR_MAX <= out.remaining())
+ {
+ auto n = std::wcrtomb(out.current(), codePoint,
&this->state);
+ if (static_cast<std::size_t>(-1) == n) // not a
valid wide character?
+ result = APR_BADARG;
+ else
+ out.increment_position(n);
+ }
+ else
+ result = APR_BADARG;
+ return result;
+ }
+
private:
std::mbstate_t state;
};
@@ -535,7 +649,11 @@ CharsetEncoderPtr CharsetEncoder::getDefaultEncoder()
CharsetEncoder* CharsetEncoder::createDefaultEncoder()
{
#if LOG4CXX_CHARSET_UTF8
+#if LOG4CXX_LOGCHAR_IS_UTF8
+ return new TrivialCharsetEncoder();
+#else
return new UTF8CharsetEncoder();
+#endif
#elif LOG4CXX_CHARSET_ISO88591
return new ISOLatinCharsetEncoder();
#elif LOG4CXX_CHARSET_USASCII
@@ -604,10 +722,11 @@ void CharsetEncoder::reset()
{
}
+#if LOG4CXX_ABI_VERSION <= 15
void CharsetEncoder::flush(ByteBuffer& /* out */ )
{
}
-
+#endif
void CharsetEncoder::encode(CharsetEncoderPtr& enc,
const LogString& src,
@@ -633,7 +752,32 @@ void CharsetEncoder::encode(CharsetEncoderPtr& enc,
#else
#error logchar is unrecognized
#endif
- dst.put(Transcoder::LOSSCHAR);
+#if 15 < LOG4CXX_ABI_VERSION
+ enc->encode(Transcoder::LOSSCHAR, dst);
+#else // LOG4CXX_ABI_VERSION <= 15
+ if (auto p = dynamic_cast<TrivialCharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+ else if (auto p = dynamic_cast<UTF8CharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+ else if (auto p =
dynamic_cast<LocaleCharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+ else if (auto p =
dynamic_cast<USASCIICharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+ else if (auto p =
dynamic_cast<ISOLatinCharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+ else if (auto p =
dynamic_cast<UTF16BECharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+ else if (auto p =
dynamic_cast<UTF16LECharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS
+ else if (auto p =
dynamic_cast<WcstombsCharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+#endif // LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS
+#if APR_HAS_XLATE
+ else if (auto p = dynamic_cast<APRCharsetEncoder*>(enc.get()))
+ p->encode(Transcoder::LOSSCHAR, dst);
+#endif // APR_HAS_XLATE
+#endif // LOG4CXX_ABI_VERSION <= 15
}
}
diff --git a/src/main/cpp/jsonlayout.cpp b/src/main/cpp/jsonlayout.cpp
index 6f5e1b86..a66c8a77 100644
--- a/src/main/cpp/jsonlayout.cpp
+++ b/src/main/cpp/jsonlayout.cpp
@@ -223,27 +223,10 @@ void JSONLayout::appendItem(const LogString& input,
LogString& buf)
for (auto nextCodePoint = start; input.end() != nextCodePoint; )
{
auto lastCodePoint = nextCodePoint;
- auto ch = Transcoder::decode(input, nextCodePoint);
- if (nextCodePoint == lastCodePoint) // failed to decode input?
- {
- // Skip the undecodable run and keep escaping the
remaining input
- // instead of discarding it; the run collapses to one
replacement.
- for (++nextCodePoint; nextCodePoint != input.end();
++nextCodePoint)
- {
- auto probe = nextCodePoint;
- Transcoder::decode(input, probe);
- if (probe != nextCodePoint) // next unit starts
a decodable sequence
- break;
- }
- ch = 0xFFFD; // The Unicode replacement character
- }
- else if ((0xD800 <= ch && ch <= 0xDFFF) || 0x10FFFF < ch)
- {
- ch = 0xFFFD; // The Unicode replacement character
- }
- else if (0x22 == ch || 0x5c == ch) // double quote or backslash?
+ auto ch = Transcoder::getCodePoint(input, nextCodePoint);
+ if (0x22 == ch || 0x5c == ch) // double quote or backslash?
;
- else if (0x20 <= ch) // not a control character?
+ else if (0x20 <= ch && 0xFFFD != ch) // not a control character
or the replacement character?
continue;
if (start != lastCodePoint)
diff --git a/src/main/cpp/outputstreamwriter.cpp
b/src/main/cpp/outputstreamwriter.cpp
index fbbbd222..51d0f427 100644
--- a/src/main/cpp/outputstreamwriter.cpp
+++ b/src/main/cpp/outputstreamwriter.cpp
@@ -118,7 +118,6 @@ void OutputStreamWriter::write(
LOG4CXX_WRITE_WRITER_FORMAL_PARAMETERS )
}
CharsetEncoder::encode(m_priv->enc, str, iter, buf);
- m_priv->enc->flush(buf);
buf.flip();
m_priv->out->write(buf);
}
diff --git a/src/main/cpp/telnetappender.cpp b/src/main/cpp/telnetappender.cpp
index 96845a26..9dd071ba 100644
--- a/src/main/cpp/telnetappender.cpp
+++ b/src/main/cpp/telnetappender.cpp
@@ -255,7 +255,7 @@ void TelnetAppender::write(ByteBuffer& buf)
void TelnetAppender::writeStatus(const SocketPtr& socket, const LogString&
msg, Pool& p)
{
- size_t bytesSize = msg.size() * 2;
+ size_t bytesSize = msg.size() * 2 + 10;
char* bytes = p.pstralloc(bytesSize);
LogString::const_iterator msgIter(msg.begin());
@@ -263,7 +263,7 @@ void TelnetAppender::writeStatus(const SocketPtr& socket,
const LogString& msg,
while (msgIter != msg.end())
{
- _priv->encoder->encode(msg, msgIter, buf);
+ CharsetEncoder::encode(_priv->encoder, msg, msgIter, buf);
buf.flip();
socket->write(buf);
buf.clear();
@@ -282,7 +282,7 @@ void TelnetAppender::append(
LOG4CXX_APPEND_FORMAL_PARAMETERS )
else
msg = event->getRenderedMessage();
msg.append(LOG4CXX_STR("\r\n"));
- size_t bytesSize = msg.size() * 2;
+ size_t bytesSize = msg.size() * 2 + 10;
char* bytes = tempPool.pstralloc(bytesSize);
LogString::const_iterator msgIter(msg.begin());
@@ -292,21 +292,10 @@ void TelnetAppender::append(
LOG4CXX_APPEND_FORMAL_PARAMETERS )
while (msgIter != msg.end())
{
- log4cxx_status_t stat = _priv->encoder->encode(msg,
msgIter, buf);
+ CharsetEncoder::encode(_priv->encoder, msg, msgIter,
buf);
buf.flip();
write(buf);
buf.clear();
-
- if (CharsetEncoder::isError(stat))
- {
- LogString unrepresented(1, 0x3F /* '?' */);
- LogString::const_iterator
unrepresentedIter(unrepresented.begin());
- stat = _priv->encoder->encode(unrepresented,
unrepresentedIter, buf);
- buf.flip();
- write(buf);
- buf.clear();
- msgIter++;
- }
}
}
}
diff --git a/src/main/cpp/transcoder.cpp b/src/main/cpp/transcoder.cpp
index 12a0c568..bf71a302 100644
--- a/src/main/cpp/transcoder.cpp
+++ b/src/main/cpp/transcoder.cpp
@@ -39,34 +39,14 @@
using namespace LOG4CXX_NS;
using namespace LOG4CXX_NS::helpers;
-
void Transcoder::decodeUTF8(const std::string& src, LogString& dst)
{
std::string::const_iterator iter = src.begin();
while (iter != src.end())
{
- std::string::const_iterator start = iter;
- unsigned int sv = decode(src, iter);
-
- if (sv != 0xFFFF)
- {
- encode(sv, dst);
- }
- else
- {
- dst.append(1, LOSSCHAR);
-
- // decode() returns 0xFFFF both for a decode error
(iter left at
- // start) and for a successfully decoded U+FFFF (iter
already
- // advanced past EF BF BF). Only advance here in the
former case,
- // otherwise the byte following U+FFFF is skipped and,
at end of
- // input, iter is pushed past src.end().
- if (iter == start)
- {
- iter++;
- }
- }
+ auto sv = getCodePoint(src, iter);
+ encode(sv, dst);
}
}
@@ -79,17 +59,8 @@ void Transcoder::encodeUTF8(const LogString& src,
std::string& dst)
while (iter != src.end())
{
- unsigned int sv = decode(src, iter);
-
- if (sv != 0xFFFF)
- {
- encode(sv, dst);
- }
- else
- {
- dst.append(1, LOSSCHAR);
- iter++;
- }
+ unsigned int sv = getCodePoint(src, iter);
+ encode(sv, dst);
}
#endif
@@ -226,6 +197,36 @@ unsigned int Transcoder::decode(const std::string& src,
return result;
}
+ template <typename T>
+unsigned int decodeCodePoint(const typename std::basic_string<T>& str,
typename std::basic_string<T>::const_iterator& nextCodePoint)
+{
+ auto lastCodePoint = nextCodePoint;
+ auto ch = Transcoder::decode(str, nextCodePoint);
+ if (nextCodePoint == lastCodePoint) // failed to decode input?
+ {
+ // Skip the undecodable run and keep escaping the remaining
input
+ // instead of discarding it; the run collapses to one
replacement.
+ for (++nextCodePoint; nextCodePoint != str.end();
++nextCodePoint)
+ {
+ auto probe = nextCodePoint;
+ Transcoder::decode(str, probe);
+ if (probe != nextCodePoint) // next unit starts a
decodable sequence
+ break;
+ }
+ ch = 0xFFFD; // The Unicode replacement character
+ }
+ else if ( (0xD800 <= ch && ch <= 0xDFFF) // UTF-16 surrogate-range
+ || 0xFFFF == ch || 0x10FFFF < ch)
+ {
+ ch = 0xFFFD; // The Unicode replacement character
+ }
+ return ch;
+}
+
+unsigned int Transcoder::getCodePoint(const std::string& str,
std::string::const_iterator& nextCodePoint)
+{
+ return decodeCodePoint<char>(str, nextCodePoint);
+}
void Transcoder::encode(unsigned int sv, std::string& dst)
{
@@ -234,6 +235,11 @@ void Transcoder::encode(unsigned int sv, std::string& dst)
dst.append(tmp, bytes);
}
+/// Does \c str contain the Unicode replacement character
+bool Transcoder::hasReplacementCharacter(const std::string& str)
+{
+ return str.npos != str.find("\xEF\xBF\xBD");
+}
void Transcoder::decode(const std::string& src, LogString& dst)
{
@@ -265,7 +271,7 @@ void Transcoder::decode(const std::string& src, LogString&
dst)
if (CharsetDecoder::isError(stat))
{
- dst.append(1, LOSSCHAR);
+ encode(LOSSCHAR, dst);
buf.increment_position(1);
}
}
@@ -323,7 +329,7 @@ void Transcoder::encode(const LogString& src, std::string&
dst)
if (CharsetEncoder::isError(stat))
{
- dst.append(1, LOSSCHAR);
+ encode(LOSSCHAR, dst);
iter++;
}
}
@@ -413,21 +419,8 @@ void Transcoder::decode(const std::wstring& src,
LogString& dst)
while (i != src.end())
{
- std::wstring::const_iterator start = i;
- unsigned int cp = decode(src, i);
-
- if (cp != 0xFFFF)
- {
- encode(cp, dst);
- }
- else
- {
- dst.append(1, LOSSCHAR);
- if (i == start)
- {
- i++;
- }
- }
+ auto cp = getCodePoint(src, i);
+ encode(cp, dst);
}
#endif
@@ -441,21 +434,8 @@ void Transcoder::encode(const LogString& src,
std::wstring& dst)
for (LogString::const_iterator i = src.begin(); i != src.end();)
{
- LogString::const_iterator start = i;
- unsigned int cp = Transcoder::decode(src, i);
-
- if (cp != 0xFFFF)
- {
- encode(cp, dst);
- }
- else
- {
- dst.append(1, LOSSCHAR);
- if (i == start)
- {
- i++;
- }
- }
+ unsigned int cp = getCodePoint(src, i);
+ encode(cp, dst);
}
#endif
@@ -487,6 +467,10 @@ unsigned int Transcoder::decode(const std::wstring& in,
#endif
}
+unsigned int Transcoder::getCodePoint(const std::wstring& str,
std::wstring::const_iterator& nextCodePoint)
+{
+ return decodeCodePoint<wchar_t>(str, nextCodePoint);
+}
void Transcoder::encode(unsigned int sv, std::wstring& dst)
{
@@ -506,6 +490,11 @@ void Transcoder::encode(unsigned int sv, std::wstring& dst)
#endif
}
+/// Does \c str contain the Unicode replacement character
+bool Transcoder::hasReplacementCharacter(const std::wstring& str)
+{
+ return str.npos != str.find(LOSSCHAR);
+}
#endif
@@ -529,7 +518,7 @@ void Transcoder::decode(const std::basic_string<UniChar>&
src, LogString& dst)
}
else
{
- dst.append(1, LOSSCHAR);
+ encode(LOSSCHAR, dst);
if (i == start)
{
i++;
@@ -575,11 +564,21 @@ unsigned int Transcoder::decode(const
std::basic_string<UniChar>& in,
return decodeUTF16(in, iter);
}
+unsigned int Transcoder::getCodePoint(const std::basic_string<UniChar>& str,
std::basic_string<UniChar>::const_iterator& nextCodePoint)
+{
+ return decodeCodePoint<UniChar>(str, nextCodePoint);
+}
+
void Transcoder::encode(unsigned int sv, std::basic_string<UniChar>& dst)
{
encodeUTF16(sv, dst);
}
+/// Does \c str contain the Unicode replacement character
+bool Transcoder::hasReplacementCharacter(const std::basic_string<UniChar>& str)
+{
+ return str.npos != str.find(LOSSCHAR);
+}
#endif
#if LOG4CXX_CFSTRING_API
@@ -610,7 +609,7 @@ void Transcoder::decode(const CFStringRef& src, LogString&
dst)
}
else
{
- dst.append(1, LOSSCHAR);
+ encode(LOSSCHAR, dst);
if (i == start)
{
i++;
@@ -672,7 +671,7 @@ std::string Transcoder::encodeCharsetName(const LogString&
val)
}
else
{
- out.append(1, LOSSCHAR);
+ out.append(1, '?');
}
}
diff --git a/src/main/cpp/transform.cpp b/src/main/cpp/transform.cpp
index e5cec926..7ede9a6f 100644
--- a/src/main/cpp/transform.cpp
+++ b/src/main/cpp/transform.cpp
@@ -43,31 +43,14 @@ void appendValidCharacters(LogString& buf, const LogString&
input, CharProcessor
for (auto nextCodePoint = start; input.end() != nextCodePoint; )
{
auto lastCodePoint = nextCodePoint;
- auto ch = Transcoder::decode(input, nextCodePoint);
- if (nextCodePoint == lastCodePoint) // failed to decode input?
- {
- // Skip the undecodable run and keep escaping the
remaining input
- // instead of discarding it; the run collapses to one
replacement.
- for (++nextCodePoint; nextCodePoint != input.end();
++nextCodePoint)
- {
- auto probe = nextCodePoint;
- Transcoder::decode(input, probe);
- if (probe != nextCodePoint) // next unit starts
a decodable sequence
- break;
- }
- }
- else if (0xD800 <= ch && ch <= 0xDFFF)
- {
- // RFC 3629 §3 explicitly forbids surrogate-half values
in UTF-8
- ch = 0xFFFF;
- }
- else if (((0x20 <= ch && ch <= 0xD7FF) &&
+ auto ch = Transcoder::getCodePoint(input, nextCodePoint);
+ if (((0x20 <= ch && ch <= 0xD7FF) &&
specials[0] != ch &&
specials[1] != ch &&
specials[2] != ch &&
specials[3] != ch) ||
(0x9 == ch || 0xA == ch || 0xD == ch) ||
- (0xE000 <= ch && ch <= 0xFFFD) ||
+ (0xE000 <= ch && ch < 0xFFFD) ||
(0x10000 <= ch && ch <= 0x10FFFF))
{
LogString escaped;
@@ -104,8 +87,8 @@ void appendValidCharacters(LogString& buf, const LogString&
input, CharProcessor
buf.append(LOG4CXX_STR(">"));
break;
- case 0xFFFF: // invalid sequence
- Transform::appendCharacterReference(buf,
0xFFFD); // The Unicode replacement character
+ case 0xFFFD: // The Unicode replacement character
+ Transform::appendCharacterReference(buf,
0xFFFD);
break;
default:
@@ -147,21 +130,8 @@ void Transform::appendEscapingCDATA(
{
bool cdataEnd = false;
auto lastCodePoint = nextCodePoint;
- auto ch = Transcoder::decode(input, nextCodePoint);
- if (nextCodePoint == lastCodePoint) // failed to decode input?
- {
- // Skip the undecodable run and keep escaping the
remaining input
- // instead of discarding it; the run collapses to one
replacement.
- for (++nextCodePoint; nextCodePoint != input.end();
++nextCodePoint)
- {
- auto probe = nextCodePoint;
- Transcoder::decode(input, probe);
- if (probe != nextCodePoint) // next unit starts
a decodable sequence
- break;
- }
- ch = 0xFFFD; // The Unicode replacement character
- }
- else if (CDATA_END[0] == ch && input.end() != nextCodePoint)
+ auto ch = Transcoder::getCodePoint(input, nextCodePoint);
+ if (CDATA_END[0] == ch && input.end() != nextCodePoint)
{
lastCodePoint = nextCodePoint;
if (CDATA_END[1] != Transcoder::decode(input,
nextCodePoint) ||
@@ -176,7 +146,7 @@ void Transform::appendEscapingCDATA(
}
else if ((0x20 <= ch && ch <= 0xD7FF) ||
(0x9 == ch || 0xA == ch || 0xD == ch) ||
- (0xE000 <= ch && ch <= 0xFFFD) ||
+ (0xE000 <= ch && ch < 0xFFFD) ||
(0x10000 <= ch && ch <= 0x10FFFF))
{
continue;
diff --git a/src/main/include/log4cxx/helpers/charsetencoder.h
b/src/main/include/log4cxx/helpers/charsetencoder.h
index e4c0b348..44b7b5dd 100644
--- a/src/main/include/log4cxx/helpers/charsetencoder.h
+++ b/src/main/include/log4cxx/helpers/charsetencoder.h
@@ -31,8 +31,8 @@ class CharsetEncoder;
LOG4CXX_PTR_DEF(CharsetEncoder);
/**
-* An engine to transform LogStrings into bytes
-* for the specific character set.
+* An abstract base for classes that transform LogString codepoints
+* to the byte sequence representation of the codepoints.
*/
class LOG4CXX_EXPORT CharsetEncoder : public Object
{
@@ -75,37 +75,46 @@ class LOG4CXX_EXPORT CharsetEncoder : public Object
static CharsetEncoderPtr getUTF8Encoder();
/**
- * Encodes a string replacing unmappable
- * characters with escape sequences.
- *
- */
+ * Use \c enc to encode into \c out codepoints of \c src until
\c out is full or an invalid value is encountered.
+ * If encoding is stopped by an invalid value,
+ * a replacement character is added to \c out
+ * and \c iter is advanced past the invalid value.
+ * @pre \c iter is a valid, dereferenceable iterator.
+ * @pre \c iter and the end of \c src are in the same sequence.
+ */
static void encode(CharsetEncoderPtr& enc,
const LogString& src,
LogString::const_iterator& iter,
ByteBuffer& dst);
/**
- * Encodes as many characters from the input string as possible
- * to the output buffer.
- * @param in input string
- * @param iter position in string to start.
- * @param out output buffer.
- * @return APR_SUCCESS unless a character can not be
represented in
- * the encoding.
+ * Encode into \c out codepoints of \c in until \c out is full
or an invalid value is encountered.
+ * @pre \c iter is a valid, dereferenceable iterator.
+ * @pre \c iter and the end of \c in are in the same sequence.
+ * @return APR_SUCCESS unless an invalid value is encountered
*/
virtual log4cxx_status_t encode(const LogString& in,
LogString::const_iterator& iter,
ByteBuffer& out) = 0;
+#if 15 < LOG4CXX_ABI_VERSION
+ /**
+ * Add onto \c out an encoded equivalent of \c codePoint.
+ * @return APR_SUCCESS unless \c codePoint cannot be
represented by this charset or \c out is full.
+ */
+ virtual log4cxx_status_t encode(unsigned int codePoint,
ByteBuffer& out) = 0;
+#endif
/**
* Resets any internal state.
*/
virtual void reset();
+#if LOG4CXX_ABI_VERSION <= 15
/**
* Flushes the encoder.
*/
virtual void flush(ByteBuffer& out);
+#endif
/**
* Determines if the return value from encode indicates
diff --git a/src/main/include/log4cxx/helpers/transcoder.h
b/src/main/include/log4cxx/helpers/transcoder.h
index 68cf8388..51ed6b96 100644
--- a/src/main/include/log4cxx/helpers/transcoder.h
+++ b/src/main/include/log4cxx/helpers/transcoder.h
@@ -33,8 +33,6 @@ class Pool;
class LOG4CXX_EXPORT Transcoder
{
public:
-
-
/**
* Append the UTF-8 characters in \c src onto \c dst.
*/
@@ -71,16 +69,30 @@ class LOG4CXX_EXPORT Transcoder
* @pre \c iter and the end of \c str are in the same
sequence.
* @param str contains the code point to which \c iter refers.
* @param iter the start of the current code point.
- * @return the code point value or 0xFFFF if not a valid
sequence.
+ * @return if a valid sequence, the decoded value; otherwise,
0xFFFF and leave \c iter unchanged.
*/
static unsigned int decode(const std::string& str,
std::string::const_iterator& iter);
+ /**
+ * Increment \c pCodePoint past one \c str code point.
+ * @pre \c pCodePoint is a valid, dereferenceable iterator.
+ * @pre \c pCodePoint and the end of \c str are in the same
sequence.
+ * @post \c <code>[old_pCodePoint = pCodePoint]
(old_pCodePoint < pCodePoint)</code> // \c pCodePoint is always advanced
+ * @param str contains the code point to which \c pCodePoint
refers.
+ * @param pCodePoint the start of the current code point.
+ * @return the code point value or 0xFFFD if not a valid
sequence; \c pCodePoint is always advanced.
+ */
+ static unsigned int getCodePoint(const std::string& str,
std::string::const_iterator& pCodePoint);
+
/**
* Append the UTF8 equivalent to \c ch onto \c dst.
*/
static void encode(unsigned int ch, std::string& dst);
+ /// Does \c str contain the Unicode replacement character
+ static bool hasReplacementCharacter(const std::string& str);
+
/**
* Append the LogString equivalent of \c src onto \c dst.
*/
@@ -126,16 +138,29 @@ class LOG4CXX_EXPORT Transcoder
* @pre \c iter and the end of \c str are in the same
sequence.
* @param str contains the code point to which \c iter refers.
* @param iter the start of the current code point.
- * @return the code point value or 0xFFFF if not a valid
sequence.
+ * @return if a valid sequence, the decoded value; otherwise,
0xFFFF and leave \c iter unchanged.
*/
static unsigned int decode(const std::wstring& str,
std::wstring::const_iterator& iter);
+ /**
+ * Increment \c pCodePoint past one \c str code point.
+ * @pre \c pCodePoint is a valid, dereferenceable iterator.
+ * @pre \c pCodePoint and the end of \c str are in the same
sequence.
+ * @post \c <code>[old_pCodePoint = pCodePoint]
(old_pCodePoint < pCodePoint)</code> // \c pCodePoint is always advanced
+ * @param str contains the code point to which \c pCodePoint
refers.
+ * @param pCodePoint the start of the current code point.
+ * @return the code point value or 0xFFFD if not a valid
sequence; \c pCodePoint is always advanced.
+ */
+ static unsigned int getCodePoint(const std::wstring& str,
std::wstring::const_iterator& pCodePoint);
+
/**
* Append the wchar_t equivalent to \c ch onto \c dst.
*/
static void encode(unsigned int ch, std::wstring& dst);
+ /// Does \c str contain the Unicode replacement character
+ static bool hasReplacementCharacter(const std::wstring& str);
#endif
@@ -155,16 +180,29 @@ class LOG4CXX_EXPORT Transcoder
* @pre \c iter and the end of \c str are in the same
sequence.
* @param str contains the code point to which \c iter refers.
* @param iter the start of the current code point.
- * @return the code point value or 0xFFFF if not a valid
sequence.
+ * @return if a valid sequence, the decoded value; otherwise,
0xFFFF and leave \c iter unchanged.
*/
static unsigned int decode(const std::basic_string<UniChar>&
str,
std::basic_string<UniChar>::const_iterator& iter);
+ /**
+ * Increment \c pCodePoint past one \c str code point.
+ * @pre \c pCodePoint is a valid, dereferenceable iterator.
+ * @pre \c pCodePoint and the end of \c str are in the same
sequence.
+ * @post \c <code>[old_pCodePoint = pCodePoint]
(old_pCodePoint < pCodePoint)</code> // \c pCodePoint is always advanced
+ * @param str contains the code point to which \c pCodePoint
refers.
+ * @param pCodePoint the start of the current code point.
+ * @return the code point value or 0xFFFD if not a valid
sequence; \c pCodePoint is always advanced.
+ */
+ static unsigned int getCodePoint(const
std::basic_string<UniChar>& str, std::basic_string<UniChar>::const_iterator&
pCodePoint);
+
/**
* Append the UniChar equivalent to \c ch onto \c dst.
*/
static void encode(unsigned int ch, std::basic_string<UniChar>&
dst);
+ /// Does \c str contain the Unicode replacement character
+ static bool hasReplacementCharacter(const
std::basic_string<UniChar>& str);
#endif
#if LOG4CXX_CFSTRING_API
@@ -176,8 +214,7 @@ class LOG4CXX_EXPORT Transcoder
static CFStringRef encode(const LogString& src);
#endif
- enum { LOSSCHAR = 0x3F };
-
+ enum { LOSSCHAR = 0xFFFD }; // Unicode replacement character
/**
* The logchar equivalent to \c ch.
*/
diff --git a/src/test/cpp/helpers/charsetencodertestcase.cpp
b/src/test/cpp/helpers/charsetencodertestcase.cpp
index 8d1430e4..db67fb44 100644
--- a/src/test/cpp/helpers/charsetencodertestcase.cpp
+++ b/src/test/cpp/helpers/charsetencodertestcase.cpp
@@ -243,8 +243,8 @@ public:
/**
* Regression test: write malformed UTF-8 through OutputStreamWriter
- * using a non-trivial encoder and assert the replacement character
- * is emitted (Transcoder::LOSSCHAR). This is deterministic and does
+ * using a non-trivial encoder and assert the encoder specific
+ * replacement character is emitted. This is deterministic and does
* not rely on process crash.
*/
void utf8Recovery()
@@ -276,8 +276,7 @@ public:
LOGUNIT_ASSERT_EQUAL(1, read);
LOGUNIT_ASSERT_EQUAL((size_t)1, buf.position());
- LOGUNIT_ASSERT_EQUAL((unsigned char) Transcoder::LOSSCHAR,
- (unsigned char) raw[0]);
+ LOGUNIT_ASSERT_EQUAL('?', raw[0]);
}
class ThreadPackage
diff --git a/src/test/cpp/helpers/transcodertestcase.cpp
b/src/test/cpp/helpers/transcodertestcase.cpp
index 18e7070c..6b808995 100644
--- a/src/test/cpp/helpers/transcodertestcase.cpp
+++ b/src/test/cpp/helpers/transcodertestcase.cpp
@@ -200,17 +200,23 @@ public:
LOGUNIT_ASSERT_EQUAL(std::wstring(L"Hello"),
encoded.substr(BUFSIZE - 3));
}
+ // Test invalid codepoints
void encode3_1()
{
- // Test invalid multibyte string
- LogString greeting;
- greeting.push_back( logchar(0xff) );
- std::wstring encoded;
- Transcoder::encode(greeting, encoded);
+ unsigned int replacementCharacter = Transcoder::LOSSCHAR;
+ std::wstring encodedD800(1, 0xD800);
+ std::wstring::const_iterator iD800 = encodedD800.begin();
+ LOGUNIT_ASSERT_EQUAL(replacementCharacter,
Transcoder::getCodePoint(encodedD800, iD800));
+
+ std::wstring encodedDFFF(1, 0xDFFF);
+ std::wstring::const_iterator iDFFF = encodedDFFF.begin();
+ LOGUNIT_ASSERT_EQUAL(replacementCharacter,
Transcoder::getCodePoint(encodedDFFF, iDFFF));
- std::wstring expected;
- expected.push_back( log4cxx::helpers::Transcoder::LOSSCHAR );
- LOGUNIT_ASSERT_EQUAL(encoded, expected );
+#if defined(__STDC_ISO_10646__)
+ std::wstring encoded110000(1, 0x110000);
+ std::wstring::const_iterator i110000 = encoded110000.begin();
+ LOGUNIT_ASSERT_EQUAL(replacementCharacter,
Transcoder::getCodePoint(encoded110000, i110000));
+#endif
}
#endif
@@ -306,7 +312,7 @@ public:
std::string src(1, char(0x80));
LogString out;
Transcoder::decodeUTF8(src, out);
- LOGUNIT_ASSERT_EQUAL(LogString(1, Transcoder::LOSSCHAR), out);
+ LOGUNIT_ASSERT(Transcoder::hasReplacementCharacter(out));
}
void testDecodeUTF8_3()
@@ -314,7 +320,7 @@ public:
std::string src("\xC2");
LogString out;
Transcoder::decodeUTF8(src, out);
- LOGUNIT_ASSERT_EQUAL(LogString(1, Transcoder::LOSSCHAR), out);
+ LOGUNIT_ASSERT(Transcoder::hasReplacementCharacter(out));
}
void testDecodeUTF8_4()
@@ -333,8 +339,7 @@ public:
* (U+D800..U+DFFF). The three-byte sequences ED A0 80 .. ED BF BF must
* not decode to the corresponding surrogate code points: doing so lets
* lone surrogates enter LogString and be re-emitted by JSON/XML
layouts,
- * propagating malformed Unicode past the parsing boundary. Each byte of
- * the invalid sequence is replaced with Transcoder::LOSSCHAR.
+ * propagating malformed Unicode past the parsing boundary.
*/
void testDecodeUTF8_RejectSurrogate()
{
@@ -342,12 +347,7 @@ public:
std::string src("\xED\xA0\x80");
LogString out;
Transcoder::decodeUTF8(src, out);
-
- LogString expected;
- expected.append(1, Transcoder::LOSSCHAR);
- expected.append(1, Transcoder::LOSSCHAR);
- expected.append(1, Transcoder::LOSSCHAR);
- LOGUNIT_ASSERT_EQUAL(expected, out);
+ LOGUNIT_ASSERT(Transcoder::hasReplacementCharacter(out));
}
/**
@@ -364,11 +364,7 @@ public:
std::string src("\xE0\xA0\x80");
LogString out;
Transcoder::decodeUTF8(src, out);
-
- LogString expected;
- Transcoder::encode(0x0800, expected);
- LOGUNIT_ASSERT_EQUAL(expected, out);
- LOGUNIT_ASSERT(out.find(Transcoder::LOSSCHAR) ==
LogString::npos);
+ LOGUNIT_ASSERT(!Transcoder::hasReplacementCharacter(out));
}
/**
@@ -392,8 +388,7 @@ public:
std::string src(c.bytes, c.len);
LogString out;
Transcoder::decodeUTF8(src, out);
- bool hasLoss = out.find(Transcoder::LOSSCHAR) !=
LogString::npos;
- LOGUNIT_ASSERT_EQUAL(c.reject, hasLoss);
+ LOGUNIT_ASSERT_EQUAL(c.reject,
Transcoder::hasReplacementCharacter(out));
}
}
@@ -412,10 +407,7 @@ public:
LogString out;
Transcoder::decodeUTF8(src, out);
- LogString expected;
- for (int i = 0; i < 4; ++i)
- expected.append(1, Transcoder::LOSSCHAR);
- LOGUNIT_ASSERT_EQUAL(expected, out);
+ LOGUNIT_ASSERT(Transcoder::hasReplacementCharacter(out));
}
/**
@@ -438,8 +430,7 @@ public:
std::string src(c.bytes, c.len);
LogString out;
Transcoder::decodeUTF8(src, out);
- bool hasLoss = out.find(Transcoder::LOSSCHAR) !=
LogString::npos;
- LOGUNIT_ASSERT_EQUAL(c.reject, hasLoss);
+ LOGUNIT_ASSERT_EQUAL(c.reject,
Transcoder::hasReplacementCharacter(out));
}
}
@@ -448,8 +439,7 @@ public:
* branch masks the lead byte with 0x07, discarding those high bits, so
* F8 BF BF BF used to slip past the U+10FFFF bound and decode to
U+3FFFF
* (and FB/FC likewise to other in-range planes) — the same aliasing
- * filter-bypass that the F5..F7 rejection guards against. Each byte of
an
- * invalid lead sequence must be replaced with Transcoder::LOSSCHAR.
+ * filter-bypass that the F5..F7 rejection guards against.
*/
void testDecodeUTF8_RejectInvalidLeadByte()
{
@@ -466,8 +456,7 @@ public:
std::string src(c.bytes, c.len);
LogString out;
Transcoder::decodeUTF8(src, out);
- bool hasLoss = out.find(Transcoder::LOSSCHAR) !=
LogString::npos;
- LOGUNIT_ASSERT_EQUAL(c.reject, hasLoss);
+ LOGUNIT_ASSERT_EQUAL(c.reject,
Transcoder::hasReplacementCharacter(out));
}
}
