This is an automated email from the ASF dual-hosted git repository.
csantanapr pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-openwhisk-package-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 87579ec Attempt to encode non-encoded messages (#296)
87579ec is described below
commit 87579ec7c135e0c5ad677ee8db7e86d626562a1e
Author: James Dubee <[email protected]>
AuthorDate: Fri Oct 12 15:26:20 2018 -0400
Attempt to encode non-encoded messages (#296)
---
provider/consumer.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/provider/consumer.py b/provider/consumer.py
index a061c86..12cfd75 100644
--- a/provider/consumer.py
+++ b/provider/consumer.py
@@ -469,6 +469,17 @@ class ConsumerProcess (Process):
return offsets
def __encodeMessageIfNeeded(self, value):
+ # let's make sure whatever data we're getting is utf-8 encoded
+ try:
+ value.decode('utf-8')
+ except UnicodeDecodeError:
+ try:
+ logging.warn('[{}] Value is not UTF-8 encoded. Attempting
encoding...'.format(self.trigger))
+ value = value.encode('utf-8')
+ except UnicodeDecodeError:
+ logging.warn('[{}] Value contains non-unicode bytes. Replacing
invalid bytes.'.format(self.trigger))
+ value = unicode(value, errors='replace').encode('utf-8')
+
if self.encodeValueAsJSON:
try:
parsed = json.loads(value)
