Thibault Delavallée (OpenERP) has proposed merging
lp:~openerp-dev/openobject-server/trunk-fetchmail-inbox-tde into
lp:openobject-server.
Requested reviews:
OpenERP Core Team (openerp)
For more details, see:
https://code.launchpad.net/~openerp-dev/openobject-server/trunk-fetchmail-inbox-tde/+merge/134266
Tools: mail.py: (E)Mail management
Refactored a bit openerp/tools about (e)mails and html.
- moved some (e)mail related functions from misc to mail
- moved html_sanitizer into mail
- added html_email_clean, to clean incoming emails (remove quotes, signature)
- added a plaintext2html function, to convert plaintext into html (\n and \r
are replaced by <br/>, more than two <br/> are replaced by end and beginning of
paragraphs)
- append_content_to_html now uses plaintext2html, according to some new flags
- added tests for html_email_clean, plaintext2html, , updated test for
append_content_to_html
--
https://code.launchpad.net/~openerp-dev/openobject-server/trunk-fetchmail-inbox-tde/+merge/134266
Your team OpenERP R&D Team is subscribed to branch
lp:~openerp-dev/openobject-server/trunk-fetchmail-inbox-tde.
=== modified file 'openerp/addons/base/base_data.xml'
--- openerp/addons/base/base_data.xml 2012-11-08 09:56:04 +0000
+++ openerp/addons/base/base_data.xml 2012-11-14 09:28:22 +0000
@@ -77,7 +77,8 @@
<field name="company_id" ref="main_company"/>
<field name="company_ids" eval="[(4, ref('main_company'))]"/>
<field name="menu_id" ref="action_menu_admin"/>
- <field name="signature">Administrator</field>
+ <field name="signature">--
+Administrator</field>
</record>
<record id="main_partner" model="res.partner">
=== modified file 'openerp/addons/base/base_demo.xml'
--- openerp/addons/base/base_demo.xml 2012-11-08 09:56:04 +0000
+++ openerp/addons/base/base_demo.xml 2012-11-14 09:28:22 +0000
@@ -11,7 +11,8 @@
<field name="partner_id" ref="base.partner_demo"/>
<field name="login">demo</field>
<field name="password">demo</field>
- <field name="signature">Mr Demo</field>
+ <field name="signature">--
+Mr Demo</field>
<field name="company_id" ref="main_company"/>
<field name="groups_id" eval="[(6,0,[ref('base.group_user')])]"/>
</record>
=== modified file 'openerp/osv/fields.py'
--- openerp/osv/fields.py 2012-11-05 13:17:12 +0000
+++ openerp/osv/fields.py 2012-11-14 09:28:22 +0000
@@ -44,8 +44,8 @@
import openerp.tools as tools
from openerp.tools.translate import _
from openerp.tools import float_round, float_repr
+from openerp.tools import html_sanitize
import simplejson
-from openerp.tools.html_sanitize import html_sanitize
from openerp import SUPERUSER_ID
_logger = logging.getLogger(__name__)
=== modified file 'openerp/osv/osv.py'
--- openerp/osv/osv.py 2012-11-06 18:15:07 +0000
+++ openerp/osv/osv.py 2012-11-14 09:28:22 +0000
@@ -120,6 +120,7 @@
raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
return f(self, dbname, *args, **kwargs)
except orm.except_orm, inst:
+ raise
raise except_osv(inst.name, inst.value)
except except_osv:
raise
=== modified file 'openerp/tests/__init__.py'
--- openerp/tests/__init__.py 2012-11-07 11:10:11 +0000
+++ openerp/tests/__init__.py 2012-11-14 09:28:22 +0000
@@ -8,7 +8,7 @@
See the :ref:`test-framework` section in the :ref:`features` list.
"""
-from . import test_expression, test_html_sanitize, test_ir_sequence, test_orm,\
+from . import test_expression, test_mail, test_ir_sequence, test_orm,\
test_fields, test_basecase, \
test_view_validation, test_uninstall, test_misc, test_db_cursor
from . import test_ir_filters
@@ -20,7 +20,7 @@
checks = [
test_expression,
- test_html_sanitize,
+ test_mail,
test_db_cursor,
test_orm,
test_fields,
=== renamed file 'openerp/tests/test_html_sanitize.py' => 'openerp/tests/test_mail.py'
--- openerp/tests/test_html_sanitize.py 2012-09-05 15:32:12 +0000
+++ openerp/tests/test_mail.py 2012-11-14 09:28:22 +0000
@@ -1,9 +1,31 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import unittest
-from openerp.tools.html_sanitize import html_sanitize
-
-test_case = """
+# This test can be run stand-alone with something like:
+# > PYTHONPATH=. python2 openerp/tests/test_misc.py
+##############################################################################
+#
+# OpenERP, Open Source Business Applications
+# Copyright (c) 2012-TODAY OpenERP S.A. <http://openerp.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+import unittest2
+from openerp.tools import html_sanitize, html_email_clean, append_content_to_html, plaintext2html
+
+HTML_SOURCE = """
<font size="2" style="color: rgb(31, 31, 31); font-family: monospace; font-variant: normal; line-height: normal; ">test1</font>
<div style="color: rgb(31, 31, 31); font-family: monospace; font-variant: normal; line-height: normal; font-size: 12px; font-style: normal; ">
<b>test2</b></div><div style="color: rgb(31, 31, 31); font-family: monospace; font-variant: normal; line-height: normal; font-size: 12px; ">
@@ -21,21 +43,160 @@
<a href="javascript:alert('malicious code')">test link</a>
"""
-class TestSanitizer(unittest.TestCase):
+TEXT_MAIL1 = """I contact you about our meeting for tomorrow. Here is the schedule I propose:
+9 AM: brainstorming about our new amazing business app</span></li>
+9.45 AM: summary
+10 AM: meeting with Fabien to present our app
+Is everything ok for you ?
+--
+Administrator"""
+
+HTML_MAIL1 = """<div>
+<font><span>I contact you about our meeting for tomorrow. Here is the schedule I propose:</span></font>
+</div>
+<div><ul>
+<li><span>9 AM: brainstorming about our new amazing business app</span></li>
+<li><span>9.45 AM: summary</span></li>
+<li><span>10 AM: meeting with Fabien to present our app</span></li>
+</ul></div>
+<div><font><span>Is everything ok for you ?</span></font></div>"""
+
+GMAIL_REPLY1_SAN = """Hello,<div><br></div><div>Ok for me. I am replying directly in gmail, without signature.</div><div><br></div><div>Kind regards,</div><div><br></div><div>Demo.<br><br><div>On Thu, Nov 8, 2012 at 5:29 PM, <span><<a href="mailto:[email protected]">[email protected]</a>></span> wrote:<br><blockquote><div>I contact you about our meeting for tomorrow. Here is the schedule I propose:</div><div><ul><li>9 AM: brainstorming about our new amazing business app</span></li></li>
+<li>9.45 AM: summary</li><li>10 AM: meeting with Fabien to present our app</li></ul></div><div>Is everything ok for you ?</div>
+<div><p>--<br>Administrator</p></div>
+
+<div><p>Log in our portal at: <a href="http://localhost:8069#action=login&db=mail_1&login=demo";>http://localhost:8069#action=login&db=mail_1&login=demo</a></p></div>
+</blockquote></div><br></div>"""
+
+THUNDERBIRD_16_REPLY1_SAN = """ <div>On 11/08/2012 05:29 PM,
+ <a href="mailto:[email protected]">[email protected]</a> wrote:<br></div>
+ <blockquote>
+ <div>I contact you about our meeting for tomorrow. Here is the
+ schedule I propose:</div>
+ <div>
+ <ul><li>9 AM: brainstorming about our new amazing business
+ app</span></li></li>
+ <li>9.45 AM: summary</li>
+ <li>10 AM: meeting with Fabien to present our app</li>
+ </ul></div>
+ <div>Is everything ok for you ?</div>
+ <div>
+ <p>--<br>
+ Administrator</p>
+ </div>
+ <div>
+ <p>Log in our portal at:
+<a href="http://localhost:8069#action=login&db=mail_1&token=rHdWcUART5PhEnJRaXjH";>http://localhost:8069#action=login&db=mail_1&token=rHdWcUART5PhEnJRaXjH</a></p>
+ </div>
+ </blockquote>
+ Ok for me. I am replying directly below your mail, using
+ Thunderbird, with a signature.<br><br>
+ Did you receive my email about my new laptop, by the way ?<br><br>
+ Raoul.<br><pre>--
+Raoul Grosbedonnée
+</pre>"""
+
+TEXT_TPL = """Salut Raoul!
+Le 28 oct. 2012 à 00:02, Raoul Grosbedon a écrit :
+
+> C'est sûr que je suis intéressé (quote)!
+
+Trouloulou pouet pouet. Je ne vais quand même pas écrire de vrais mails, non mais ho.
+
+> 2012/10/27 Bert Tartopoils :
+>> Diantre, me disè-je en envoyant un message similaire à Martine, mais comment vas-tu (quote)?
+>>
+>> A la base le contenu était un vrai mail, mais je l'ai quand même réécrit pour ce test, histoire de dire que, quand même, on ne met pas n'importe quoi ici. (quote)
+>>
+>> Et sinon bon courage pour trouver tes clefs (quote).
+>>
+>> Bert TARTOPOILS
+>> [email protected]
+>>
+>
+>
+> --
+> Raoul Grosbedon
+
+Bert TARTOPOILS
[email protected]
+"""
+
+
+class TestSanitizer(unittest2.TestCase):
+ """ Test the html sanitizer """
+ # TDE note: could be improved by actually checking the output
def test_simple(self):
x = "yop"
self.assertEqual(x, html_sanitize(x))
-
+
def test_trailing_text(self):
x = 'lala<p>yop</p>xxx'
self.assertEqual(x, html_sanitize(x))
-
+
def test_no_exception(self):
- html_sanitize(test_case)
-
+ html_sanitize(HTML_SOURCE)
+
def test_unicode(self):
html_sanitize("Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci")
+
+class TestCleaner(unittest2.TestCase):
+ """ Test the email cleaner function that filter the content of incoming emails """
+
+ def test_html_email_clean(self):
+ # Test1: reply through gmail: quote in blockquote, signature --\nAdministrator
+ new_html = html_email_clean(GMAIL_REPLY1_SAN)
+ self.assertNotIn('blockquote', new_html, 'html_email_cleaner did not remove a blockquote')
+ self.assertNotIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+ self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+ self.assertIn('Ok for me', new_html, 'html_email_cleaner erased too much content')
+
+ # Test2: reply through Tunderbird 16.0.2
+ new_html = html_email_clean(THUNDERBIRD_16_REPLY1_SAN)
+ self.assertNotIn('blockquote', new_html, 'html_email_cleaner did not remove a blockquote')
+ self.assertNotIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+ self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+ self.assertNotIn('Grosbedonn', new_html, 'html_email_cleaner did not erase the signature')
+ self.assertIn('Ok for me', new_html, 'html_email_cleaner erased too much content')
+
+ # Test3: text email
+ new_html = html_email_clean(TEXT_MAIL1)
+ self.assertIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+ self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+
+ # Test4: more complex text email
+ new_html = html_email_clean(TEXT_TPL)
+ self.assertNotIn('quote', new_html, 'html_email_cleaner did not remove correctly plaintext quotes')
+
+
+class TestAppendContentToHtml(unittest2.TestCase):
+ """ Test some of our generic utility functions about html """
+
+ def test_plaintext2html(self):
+ cases = [
+ ("First \nSecond \nThird\n \nParagraph\n\r--\nSignature paragraph", 'div',
+ "<div><p>First <br/>Second <br/>Third</p><p>Paragraph</p><p>--<br/>Signature paragraph</p></div>"),
+ ("First<p>It should be escaped</p>\nSignature", False,
+ "<p>First<p>It should be escaped</p><br/>Signature</p>")
+ ]
+ for content, container_tag, expected in cases:
+ html = plaintext2html(content, container_tag)
+ self.assertEqual(html, expected, 'text2html is broken')
+
+ def test_append_to_html(self):
+ test_samples = [
+ ('<!DOCTYPE...><HTML encoding="blah">some <b>content</b></HtMl>', '--\nYours truly', True, True, False,
+ '<!DOCTYPE...><html encoding="blah">some <b>content</b>\n<pre>--\nYours truly</pre>\n</html>'),
+ ('<!DOCTYPE...><HTML encoding="blah">some <b>content</b></HtMl>', '--\nYours truly', True, False, False,
+ '<!DOCTYPE...><html encoding="blah">some <b>content</b>\n<p>--<br/>Yours truly</p>\n</html>'),
+ ('<html><body>some <b>content</b></body></html>', '<!DOCTYPE...>\n<html><body>\n<p>--</p>\n<p>Yours truly</p>\n</body>\n</html>', False, False, False,
+ '<html><body>some <b>content</b>\n\n\n<p>--</p>\n<p>Yours truly</p>\n\n\n</body></html>'),
+ ]
+ for html, content, plaintext_flag, preserve_flag, container_tag, expected in test_samples:
+ self.assertEqual(append_content_to_html(html, content, plaintext_flag, preserve_flag, container_tag), expected, 'append_content_to_html is broken')
+
+
if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ unittest2.main()
=== modified file 'openerp/tests/test_misc.py'
--- openerp/tests/test_misc.py 2012-09-19 11:40:47 +0000
+++ openerp/tests/test_misc.py 2012-11-14 09:28:22 +0000
@@ -4,18 +4,6 @@
import unittest2
from ..tools import misc
-class append_content_to_html(unittest2.TestCase):
- """ Test some of our generic utility functions """
-
- def test_append_to_html(self):
- test_samples = [
- ('<!DOCTYPE...><HTML encoding="blah">some <b>content</b></HtMl>', '--\nYours truly', True,
- '<!DOCTYPE...><html encoding="blah">some <b>content</b>\n<pre>--\nYours truly</pre>\n</html>'),
- ('<html><body>some <b>content</b></body></html>', '<!DOCTYPE...>\n<html><body>\n<p>--</p>\n<p>Yours truly</p>\n</body>\n</html>', False,
- '<html><body>some <b>content</b>\n\n\n<p>--</p>\n<p>Yours truly</p>\n\n\n</body></html>'),
- ]
- for html, content, flag, expected in test_samples:
- self.assertEqual(misc.append_content_to_html(html,content,flag), expected, 'append_content_to_html is broken')
class test_countingstream(unittest2.TestCase):
def test_empty_stream(self):
=== modified file 'openerp/tools/__init__.py'
--- openerp/tools/__init__.py 2012-08-13 14:22:32 +0000
+++ openerp/tools/__init__.py 2012-11-14 09:28:22 +0000
@@ -33,7 +33,7 @@
from yaml_import import *
from sql import *
from float_utils import *
-from html_sanitize import *
+from mail import *
#.apidoc title: Tools
=== renamed file 'openerp/tools/html_sanitize.py' => 'openerp/tools/mail.py'
--- openerp/tools/html_sanitize.py 2012-09-05 22:57:46 +0000
+++ openerp/tools/mail.py 2012-11-14 09:28:22 +0000
@@ -19,12 +19,27 @@
#
##############################################################################
+from lxml import etree
+import cgi
+import logging
import lxml.html
+import openerp.pooler as pooler
import operator
+import random
import re
+import socket
+import threading
+import time
from openerp.loglevels import ustr
+_logger = logging.getLogger(__name__)
+
+
+#----------------------------------------------------------
+# HTML Sanitizer
+#----------------------------------------------------------
+
def html_sanitize(src):
if not src:
return src
@@ -90,3 +105,294 @@
children[-1].tail = element
else:
dest_node.append(element)
+
+
+#----------------------------------------------------------
+# HTML Cleaner
+#----------------------------------------------------------
+
+def html_email_clean(html):
+ """ html_email_clean: clean the html to display in the web client.
+ - strip email quotes (remove blockquote nodes)
+ - strip signatures (remove --\n{\n)Blahblah), by replacing <br> by
+ \n to avoid ignoring signatures converted into html
+
+ :param string html: sanitized html; tags like html or head should not
+ be present in the html string. This method therefore takes as input
+ html code coming from a sanitized source, like fields.html.
+ """
+ html = ustr(html)
+ modified_html = ''
+
+ # 1. <br[ /]> -> \n, because otherwise the tree is obfuscated
+ br_tags = re.compile(r'([<]\s*br\s*\/?[>])')
+ idx = 0
+ for item in re.finditer(br_tags, html):
+ modified_html += html[idx:item.start()] + '__BR_TAG__'
+ idx = item.end()
+ modified_html += html[idx:]
+ html = modified_html
+ # TDE note: seems to have lots of <div><br></div> in emails... needs to be checks, could be cleaned
+
+ # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre
+ root = lxml.html.fromstring(html)
+ if not len(root) and root.text is None and root.tail is None:
+ html = '<div>%s</div>' % html
+ root = lxml.html.fromstring(html)
+
+ # 2.5 remove quoted text in nodes
+ quote_tags = re.compile(r'(\n(>)+[^\n\r]*)')
+ for node in root.getiterator():
+ if not node.text:
+ continue
+ idx = 0
+ text = ''
+ for item in re.finditer(quote_tags, node.text):
+ print item
+ text += node.text[idx:item.start()]
+ idx = item.end()
+ text += node.text[idx:]
+ node.text = text
+
+ # 3. remove blockquotes
+ quotes = [el for el in root.getiterator(tag='blockquote')]
+ for node in quotes:
+ # copy the node tail into parent text
+ if node.tail:
+ parent = node.getparent()
+ parent.text = parent.text or '' + node.tail
+ # remove the node
+ node.getparent().remove(node)
+
+ # 4. strip signatures
+ signature = re.compile(r'([-]{2}[\s]?[\r\n]{1,2}[^\z]+)')
+ for elem in root.getiterator():
+ if elem.text:
+ match = re.search(signature, elem.text)
+ if match:
+ elem.text = elem.text[:match.start()] + elem.text[match.end():]
+ if elem.tail:
+ match = re.search(signature, elem.tail)
+ if match:
+ elem.tail = elem.tail[:match.start()] + elem.tail[match.end():]
+
+ # 5. \n back to <br/>
+ html = etree.tostring(root, pretty_print=True)
+ html = html.replace('__BR_TAG__', '<br />')
+
+ # 6. Misc cleaning :
+ # - ClEditor seems to love using <div><br /><div> -> replace with <br />
+ modified_html = ''
+ br_div_tags = re.compile(r'(<div>\s*<br\s*\/>\s*<\/div>)')
+ idx = 0
+ for item in re.finditer(br_div_tags, html):
+ modified_html += html[idx:item.start()] + '<br />'
+ idx = item.end()
+ modified_html += html[idx:]
+ html = modified_html
+
+ return html
+
+
+#----------------------------------------------------------
+# HTML/Text management
+#----------------------------------------------------------
+
+def html2plaintext(html, body_id=None, encoding='utf-8'):
+ """ From an HTML text, convert the HTML to plain text.
+ If @param body_id is provided then this is the tag where the
+ body (not necessarily <body>) starts.
+ """
+ ## (c) Fry-IT, www.fry-it.com, 2007
+ ## <[email protected]>
+ ## download here: http://www.peterbe.com/plog/html2plaintext
+
+ html = ustr(html)
+ tree = etree.fromstring(html, parser=etree.HTMLParser())
+
+ if body_id is not None:
+ source = tree.xpath('//*[@id=%s]' % (body_id,))
+ else:
+ source = tree.xpath('//body')
+ if len(source):
+ tree = source[0]
+
+ url_index = []
+ i = 0
+ for link in tree.findall('.//a'):
+ url = link.get('href')
+ if url:
+ i += 1
+ link.tag = 'span'
+ link.text = '%s [%s]' % (link.text, i)
+ url_index.append(url)
+
+ html = ustr(etree.tostring(tree, encoding=encoding))
+
+ html = html.replace('<strong>', '*').replace('</strong>', '*')
+ html = html.replace('<b>', '*').replace('</b>', '*')
+ html = html.replace('<h3>', '*').replace('</h3>', '*')
+ html = html.replace('<h2>', '**').replace('</h2>', '**')
+ html = html.replace('<h1>', '**').replace('</h1>', '**')
+ html = html.replace('<em>', '/').replace('</em>', '/')
+ html = html.replace('<tr>', '\n')
+ html = html.replace('</p>', '\n')
+ html = re.sub('<br\s*/?>', '\n', html)
+ html = re.sub('<.*?>', ' ', html)
+ html = html.replace(' ' * 2, ' ')
+
+ # strip all lines
+ html = '\n'.join([x.strip() for x in html.splitlines()])
+ html = html.replace('\n' * 2, '\n')
+
+ for i, url in enumerate(url_index):
+ if i == 0:
+ html += '\n\n'
+ html += ustr('[%s] %s\n') % (i + 1, url)
+
+ return html
+
+def plaintext2html(text, container_tag=False):
+ """ Convert plaintext into html. Content of the text is escaped to manage
+ html entities, using cgi.escape().
+ - all \n,\r are replaced by <br />
+ - enclose content into <p>
+ - 2 or more consecutive <br /> are considered as paragraph breaks
+
+ :param string container_tag: container of the html; by default the
+ content is embedded into a <div>
+ """
+ text = cgi.escape(ustr(text))
+
+ # 1. replace \n and \r
+ text = text.replace('\n', '<br/>')
+ text = text.replace('\r', '<br/>')
+
+ # 2-3: form paragraphs
+ idx = 0
+ final = '<p>'
+ br_tags = re.compile(r'(([<]\s*[bB][rR]\s*\/?[>]\s*){2,})')
+ for item in re.finditer(br_tags, text):
+ final += text[idx:item.start()] + '</p><p>'
+ idx = item.end()
+ final += text[idx:] + '</p>'
+
+ # 4. container
+ if container_tag:
+ final = '<%s>%s</%s>' % (container_tag, final, container_tag)
+ return ustr(final)
+
+def append_content_to_html(html, content, plaintext=True, preserve=False, container_tag=False):
+ """ Append extra content at the end of an HTML snippet, trying
+ to locate the end of the HTML document (</body>, </html>, or
+ EOF), and converting the provided content in html unless ``plaintext``
+ is False.
+ Content conversion can be done in two ways:
+ - wrapping it into a pre (preserve=True)
+ - use plaintext2html (preserve=False, using container_tag to wrap the
+ whole content)
+ A side-effect of this method is to coerce all HTML tags to
+ lowercase in ``html``, and strip enclosing <html> or <body> tags in
+ content if ``plaintext`` is False.
+
+ :param str html: html tagsoup (doesn't have to be XHTML)
+ :param str content: extra content to append
+ :param bool plaintext: whether content is plaintext and should
+ be wrapped in a <pre/> tag.
+ :param bool preserve: if content is plaintext, wrap it into a <pre>
+ instead of converting it into html
+ """
+ html = ustr(html)
+ if plaintext and preserve:
+ content = u'\n<pre>%s</pre>\n' % ustr(content)
+ elif plaintext:
+ content = '\n%s\n' % plaintext2html(content, container_tag)
+ else:
+ content = re.sub(r'(?i)(</?html.*>|</?body.*>|<!\W*DOCTYPE.*>)', '', content)
+ content = u'\n%s\n' % ustr(content)
+ # Force all tags to lowercase
+ html = re.sub(r'(</?)\W*(\w+)([ >])',
+ lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html)
+ insert_location = html.find('</body>')
+ if insert_location == -1:
+ insert_location = html.find('</html>')
+ if insert_location == -1:
+ return '%s%s' % (html, content)
+ return '%s%s%s' % (html[:insert_location], content, html[insert_location:])
+
+#----------------------------------------------------------
+# Emails
+#----------------------------------------------------------
+
+email_re = re.compile(r"""
+ ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part
+ @ # mandatory @ sign
+ [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then?
+ \.
+ [a-z]{2,3} # TLD
+ )
+ """, re.VERBOSE)
+res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE)
+command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE)
+
+# Updated in 7.0 to match the model name as well
+# Typical form of references is <timestamp-openerp-record_id-model_name@domain>
+# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain
+reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE)
+
+def generate_tracking_message_id(res_id):
+ """Returns a string that can be used in the Message-ID RFC822 header field
+
+ Used to track the replies related to a given object thanks to the "In-Reply-To"
+ or "References" fields that Mail User Agents will set.
+ """
+ try:
+ rnd = random.SystemRandom().random()
+ except NotImplementedError:
+ rnd = random.random()
+ rndstr = ("%.15f" % rnd)[2:]
+ return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname())
+
+def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False,
+ attachments=None, message_id=None, references=None, openobject_id=False, debug=False, subtype='plain', headers=None,
+ smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None):
+ """Low-level function for sending an email (deprecated).
+
+ :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead.
+ :param email_from: A string used to fill the `From` header, if falsy,
+ config['email_from'] is used instead. Also used for
+ the `Reply-To` header if `reply_to` is not provided
+ :param email_to: a sequence of addresses to send the mail to.
+ """
+
+ # If not cr, get cr from current thread database
+ if not cr:
+ db_name = getattr(threading.currentThread(), 'dbname', None)
+ if db_name:
+ cr = pooler.get_db_only(db_name).cursor()
+ else:
+ raise Exception("No database cursor found, please pass one explicitly")
+
+ # Send Email
+ try:
+ mail_server_pool = pooler.get_pool(cr.dbname).get('ir.mail_server')
+ res = False
+ # Pack Message into MIME Object
+ email_msg = mail_server_pool.build_email(email_from, email_to, subject, body, email_cc, email_bcc, reply_to,
+ attachments, message_id, references, openobject_id, subtype, headers=headers)
+
+ res = mail_server_pool.send_email(cr, uid or 1, email_msg, mail_server_id=None,
+ smtp_server=smtp_server, smtp_port=smtp_port, smtp_user=smtp_user, smtp_password=smtp_password,
+ smtp_encryption=('ssl' if ssl else None), smtp_debug=debug)
+ except Exception:
+ _logger.exception("tools.email_send failed to deliver email")
+ return False
+ finally:
+ cr.close()
+ return res
+
+def email_split(text):
+ """ Return a list of the email addresses found in ``text`` """
+ if not text:
+ return []
+ return re.findall(r'([^ ,<@]+@[^> ,]+)', text)
=== modified file 'openerp/tools/misc.py'
--- openerp/tools/misc.py 2012-10-24 16:15:27 +0000
+++ openerp/tools/misc.py 2012-11-14 09:28:22 +0000
@@ -30,8 +30,6 @@
import subprocess
import logging
import os
-import random
-import re
import socket
import sys
import threading
@@ -48,8 +46,6 @@
except ImportError:
html2text = None
-import openerp.loglevels as loglevels
-import openerp.pooler as pooler
from config import config
from cache import *
@@ -275,168 +271,6 @@
return izip(xrange(len(l)-1, -1, -1), reversed(l))
#----------------------------------------------------------
-# Emails
-#----------------------------------------------------------
-email_re = re.compile(r"""
- ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part
- @ # mandatory @ sign
- [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then?
- \.
- [a-z]{2,3} # TLD
- )
- """, re.VERBOSE)
-res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE)
-command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE)
-
-# Updated in 7.0 to match the model name as well
-# Typical form of references is <timestamp-openerp-record_id-model_name@domain>
-# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain
-reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE)
-
-def html2plaintext(html, body_id=None, encoding='utf-8'):
- """ From an HTML text, convert the HTML to plain text.
- If @param body_id is provided then this is the tag where the
- body (not necessarily <body>) starts.
- """
- ## (c) Fry-IT, www.fry-it.com, 2007
- ## <[email protected]>
- ## download here: http://www.peterbe.com/plog/html2plaintext
-
- html = ustr(html)
-
- from lxml.etree import tostring, fromstring, HTMLParser
- tree = fromstring(html, parser=HTMLParser())
-
- if body_id is not None:
- source = tree.xpath('//*[@id=%s]'%(body_id,))
- else:
- source = tree.xpath('//body')
- if len(source):
- tree = source[0]
-
- url_index = []
- i = 0
- for link in tree.findall('.//a'):
- url = link.get('href')
- if url:
- i += 1
- link.tag = 'span'
- link.text = '%s [%s]' % (link.text, i)
- url_index.append(url)
-
- html = ustr(tostring(tree, encoding=encoding))
-
- html = html.replace('<strong>','*').replace('</strong>','*')
- html = html.replace('<b>','*').replace('</b>','*')
- html = html.replace('<h3>','*').replace('</h3>','*')
- html = html.replace('<h2>','**').replace('</h2>','**')
- html = html.replace('<h1>','**').replace('</h1>','**')
- html = html.replace('<em>','/').replace('</em>','/')
- html = html.replace('<tr>', '\n')
- html = html.replace('</p>', '\n')
- html = re.sub('<br\s*/?>', '\n', html)
- html = re.sub('<.*?>', ' ', html)
- html = html.replace(' ' * 2, ' ')
-
- # strip all lines
- html = '\n'.join([x.strip() for x in html.splitlines()])
- html = html.replace('\n' * 2, '\n')
-
- for i, url in enumerate(url_index):
- if i == 0:
- html += '\n\n'
- html += ustr('[%s] %s\n') % (i+1, url)
-
- return html
-
-def generate_tracking_message_id(res_id):
- """Returns a string that can be used in the Message-ID RFC822 header field
-
- Used to track the replies related to a given object thanks to the "In-Reply-To"
- or "References" fields that Mail User Agents will set.
- """
- try:
- rnd = random.SystemRandom().random()
- except NotImplementedError:
- rnd = random.random()
- rndstr = ("%.15f" % rnd)[2:]
- return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname())
-
-def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False,
- attachments=None, message_id=None, references=None, openobject_id=False, debug=False, subtype='plain', headers=None,
- smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None):
- """Low-level function for sending an email (deprecated).
-
- :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead.
- :param email_from: A string used to fill the `From` header, if falsy,
- config['email_from'] is used instead. Also used for
- the `Reply-To` header if `reply_to` is not provided
- :param email_to: a sequence of addresses to send the mail to.
- """
-
- # If not cr, get cr from current thread database
- if not cr:
- db_name = getattr(threading.currentThread(), 'dbname', None)
- if db_name:
- cr = pooler.get_db_only(db_name).cursor()
- else:
- raise Exception("No database cursor found, please pass one explicitly")
-
- # Send Email
- try:
- mail_server_pool = pooler.get_pool(cr.dbname).get('ir.mail_server')
- res = False
- # Pack Message into MIME Object
- email_msg = mail_server_pool.build_email(email_from, email_to, subject, body, email_cc, email_bcc, reply_to,
- attachments, message_id, references, openobject_id, subtype, headers=headers)
-
- res = mail_server_pool.send_email(cr, uid or 1, email_msg, mail_server_id=None,
- smtp_server=smtp_server, smtp_port=smtp_port, smtp_user=smtp_user, smtp_password=smtp_password,
- smtp_encryption=('ssl' if ssl else None), smtp_debug=debug)
- except Exception:
- _logger.exception("tools.email_send failed to deliver email")
- return False
- finally:
- cr.close()
- return res
-
-def email_split(text):
- """ Return a list of the email addresses found in ``text`` """
- if not text: return []
- return re.findall(r'([^ ,<@]+@[^> ,]+)', text)
-
-def append_content_to_html(html, content, plaintext=True):
- """Append extra content at the end of an HTML snippet, trying
- to locate the end of the HTML document (</body>, </html>, or
- EOF), and wrapping the provided content in a <pre/> block
- unless ``plaintext`` is False. A side-effect of this
- method is to coerce all HTML tags to lowercase in ``html``,
- and strip enclosing <html> or <body> tags in content if
- ``plaintext`` is False.
-
- :param str html: html tagsoup (doesn't have to be XHTML)
- :param str content: extra content to append
- :param bool plaintext: whether content is plaintext and should
- be wrapped in a <pre/> tag.
- """
- html = ustr(html)
- if plaintext:
- content = u'\n<pre>%s</pre>\n' % ustr(content)
- else:
- content = re.sub(r'(?i)(</?html.*>|</?body.*>|<!\W*DOCTYPE.*>)', '', content)
- content = u'\n%s\n'% ustr(content)
- # Force all tags to lowercase
- html = re.sub(r'(</?)\W*(\w+)([ >])',
- lambda m: '%s%s%s' % (m.group(1),m.group(2).lower(),m.group(3)), html)
- insert_location = html.find('</body>')
- if insert_location == -1:
- insert_location = html.find('</html>')
- if insert_location == -1:
- return '%s%s' % (html, content)
- return '%s%s%s' % (html[:insert_location], content, html[insert_location:])
-
-
-#----------------------------------------------------------
# SMS
#----------------------------------------------------------
# text must be latin-1 encoded
_______________________________________________
Mailing list: https://launchpad.net/~openerp-dev-gtk
Post to : [email protected]
Unsubscribe : https://launchpad.net/~openerp-dev-gtk
More help : https://help.launchpad.net/ListHelp
