Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package urlwatch for openSUSE:Factory checked in at 2021-12-13 20:45:15 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/urlwatch (Old) and /work/SRC/openSUSE:Factory/.urlwatch.new.2520 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "urlwatch" Mon Dec 13 20:45:15 2021 rev:22 rq:940291 version:2.24 Changes: -------- --- /work/SRC/openSUSE:Factory/urlwatch/urlwatch.changes 2021-04-14 10:11:45.585572782 +0200 +++ /work/SRC/openSUSE:Factory/.urlwatch.new.2520/urlwatch.changes 2021-12-13 20:51:34.076679890 +0100 @@ -1,0 +2,22 @@ +Mon Dec 13 16:06:55 UTC 2021 - Michael Vetter <[email protected]> + +- Update to 2.24: + Added: + * The Telegram reporter has gained two new options: + silent: Receive message notification without sound + monospace: Format message in monospace style + * Support for running a subset of jobs by specifying their index + on the command line + Changed: + * Migrated CI pipeline from Travis CI to Github Actions + * user_visible_url can now be specified for all job types (#654) + * Added a remove-duplicate-lines filter. + * Added a csv2text filter. + * Set envelope from (-f option) when sending emails using sendmail + * It is now possible to override the HTTP method when data is set + on a URL job + Fixed: + * Fix UnboundLocalError when SMTP auth is enabled, but keyring + is not installed (#667) + +------------------------------------------------------------------- Old: ---- urlwatch-2.23.tar.gz New: ---- urlwatch-2.24.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ urlwatch.spec ++++++ --- /var/tmp/diff_new_pack.Wyz0n8/_old 2021-12-13 20:51:34.560680200 +0100 +++ /var/tmp/diff_new_pack.Wyz0n8/_new 2021-12-13 20:51:34.564680202 +0100 @@ -17,7 +17,7 @@ Name: urlwatch -Version: 2.23 +Version: 2.24 Release: 0 Summary: A tool for monitoring webpages for updates License: BSD-3-Clause ++++++ urlwatch-2.23.tar.gz -> urlwatch-2.24.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/.github/workflows/unit-tests.yml new/urlwatch-2.24/.github/workflows/unit-tests.yml --- old/urlwatch-2.23/.github/workflows/unit-tests.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/urlwatch-2.24/.github/workflows/unit-tests.yml 2021-11-07 09:34:54.000000000 +0100 @@ -0,0 +1,37 @@ +name: Unit Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + name: Python + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --requirement requirements.txt + python -m pip install pycodestyle==2.6.0 pytest + sudo apt install -y build-essential libpoppler-cpp-dev pkg-config python-dev tesseract-ocr + python -m pip install pdftotext docutils pygments pytesseract pillow jq + - name: Test with pytest + run: pytest -v diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/.travis.yml new/urlwatch-2.24/.travis.yml --- old/urlwatch-2.23/.travis.yml 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/.travis.yml 1970-01-01 01:00:00.000000000 +0100 @@ -1,12 +0,0 @@ -language: python -cache: pip -python: - - "3.6" - - "3.7" - - "3.8" -install: - - pip install -r requirements.txt - - pip install pycodestyle==2.6.0 pytest - - sudo apt-get install -y build-essential libpoppler-cpp-dev pkg-config python-dev tesseract-ocr - - pip install pdftotext docutils pygments pytesseract pillow jq -script: pytest -v diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/CHANGELOG.md new/urlwatch-2.24/CHANGELOG.md --- old/urlwatch-2.23/CHANGELOG.md 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/CHANGELOG.md 2021-11-07 09:34:54.000000000 +0100 @@ -4,6 +4,28 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [2.24] -- 2021-11-07 + +### Added + +- The Telegram reporter has gained two new options: + - `silent`: Receive message notification without sound + - `monospace`: Format message in monospace style +- Support for running a subset of jobs by specifying their index on the command line + +### Changed + +- Migrated CI pipeline from Travis CI to Github Actions +- `user_visible_url` can now be specified for all job types (#654, by kongomongo) +- Added a `remove-duplicate-lines` filter. +- Added a `csv2text` filter. +- Set envelope from (`-f` option) when sending emails using `sendmail` +- It is now possible to override the HTTP `method` when `data` is set on a URL job + +### Fixed + +- Fix UnboundLocalError when SMTP auth is enabled, but keyring is not installed (#667) + ## [2.23] -- 2021-04-10 ### Added diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/README.md new/urlwatch-2.24/README.md --- old/urlwatch-2.23/README.md 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/README.md 2021-11-07 09:34:54.000000000 +0100 @@ -1,4 +1,4 @@ -[](https://travis-ci.org/thp/urlwatch) +[](https://github.com/thp/urlwatch/actions/workflows/unit-tests.yml) [](https://repology.org/metapackage/urlwatch/versions) [](https://badge.fury.io/py/urlwatch) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/docs/source/advanced.rst new/urlwatch-2.24/docs/source/advanced.rst --- old/urlwatch-2.23/docs/source/advanced.rst 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/docs/source/advanced.rst 2021-11-07 09:34:54.000000000 +0100 @@ -312,3 +312,49 @@ filter: - grep: "Thing B" + +Running a subset of jobs +------------------------ + +To run one or more specific jobs instead of all known jobs, provide +the job index numbers to the urlwatch command. For example, to run +jobs with index 2, 4, and 7: + +.. code-block:: bash + + urlwatch 2 4 7 + + +Sending HTML form data using POST +--------------------------------- + +To simulate submitting a HTML form using the POST method, you can pass +the form fields in the ``data`` field of the job description: + +.. code-block:: yaml + + name: "My POST Job" + url: http://example.com/foo + data: + username: "foo" + password: "bar" + submit: "Send query" + +By default, the request will use the HTTP ``POST`` method, and the +``Content-type`` will be set to ``application/x-www-form-urlencoded``. + + +Sending arbitrary data using HTTP PUT +------------------------------------- + +It is possible to customize the HTTP method and ``Content-type`` header, +allowing you to send arbitrary requests to the server: + +.. code-block:: yaml + + name: "My PUT Request" + url: http://example.com/item/new + method: PUT + headers: + Content-type: application/json + data: '{"foo": true}' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/docs/source/conf.py new/urlwatch-2.24/docs/source/conf.py --- old/urlwatch-2.23/docs/source/conf.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/docs/source/conf.py 2021-11-07 09:34:54.000000000 +0100 @@ -22,7 +22,7 @@ author = 'Thomas Perl' # The full version, including alpha/beta/rc tags -release = '2.23' +release = '2.24' # -- General configuration --------------------------------------------------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/docs/source/filters.rst new/urlwatch-2.24/docs/source/filters.rst --- old/urlwatch-2.23/docs/source/filters.rst 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/docs/source/filters.rst 2021-11-07 09:34:54.000000000 +0100 @@ -69,6 +69,7 @@ - **sha1sum**: Calculate the SHA-1 checksum of the content - **shellpipe**: Filter using a shell command - **sort**: Sort input items +- **remove-duplicate-lines**: Remove duplicate lines (case sensitive) - **strip**: Strip leading and trailing whitespace - **xpath**: Filter XML/HTML using XPath expressions - **jq**: Filter, transform and extract values from JSON @@ -135,9 +136,7 @@ To filter based on an `XPath <https://www.w3.org/TR/1999/REC-xpath-19991116/>`__ expression, -you can use the ``xpath`` filter like so (see Microsoft???s `XPath -Examples <https://msdn.microsoft.com/en-us/library/ms256086(v=vs.110).aspx>`__ -page for some other examples): +you can use the ``xpath`` filter like so: .. code:: yaml @@ -149,6 +148,9 @@ element, which in turn must be below the ``<html>`` element of the document, stripping out everything else. +See Microsoft???s `XPath Examples <https://msdn.microsoft.com/en-us/library/ms256086(v=vs.110).aspx>`__ page for some other examples. +You can also find an XPath of an ``<html>`` node in the Chromium/Google Chrome developer tools by right clicking on the node and selecting ``copy XPath``. + Filtering based on CSS selectors -------------------------------- @@ -301,6 +303,25 @@ password: urlwatchsecret - strip +Dealing with CSV input +---------------------- + +`csv2text` filter can be used to turn CSV data to a prettier textual representation. This is done by +supplying a `format_string` which is a [python format string](https://docs.python.org/3/library/string.html#format-string-syntax). +If the CSV has a header, the format string should use the header names (**lowercased**). Example: + +| Name | Company | +|--|--| +| Smith | Initech | +| Doe | Initech | + +Format string for the above CSV: `Mr {name} works at {company}` (Note the lowercase). +If there is no header row, you will need to use the numeric array notation: +`Mr {0} works at {1}`. +You can also use numeric array on CSV with headers with the flag `ignore_header`. +`has_header` can be used to force use the first line or first ignore the first line as header, +otherwise [csv.Sniffer](https://docs.python.org/3/library/csv.html#csv.Sniffer) +would be used. Sorting of webpage content -------------------------- @@ -396,6 +417,13 @@ - xpath: (//a[contains(@class,"item-title ref-name")])[1] - html2text +Alternatively, ``jq`` can be used for filtering: + +.. code:: yaml + + url: https://api.github.com/repos/voxpupuli/puppet-rundeck/tags + filter: + - jq: '.[0].name' Remove or replace text using regular expressions ------------------------------------------------ @@ -566,4 +594,4 @@ Supports aggregations, selections, and the built-in operators like ``length``. For more information on the operations permitted, see the `jq Manual`_. -.. _jq Manual: https://stedolan.github.io/jq/manual/ \ No newline at end of file +.. _jq Manual: https://stedolan.github.io/jq/manual/ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/docs/source/jobs.rst new/urlwatch-2.24/docs/source/jobs.rst --- old/urlwatch-2.23/docs/source/jobs.rst 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/docs/source/jobs.rst 2021-11-07 09:34:54.000000000 +0100 @@ -7,7 +7,9 @@ The list of jobs to run are contained in the configuration file ``urls.yaml``, accessed with the command ``urlwatch --edit``, each separated by a line -containing only ``---``. +containing only ``---``. The command ``urlwatch --list`` prints the name +of each job, along with its index number (1,2,3,...) which gets assigned +automatically according to its position in the configuration file. While optional, it is recommended that each job starts with a ``name`` entry: @@ -46,7 +48,6 @@ - ``ignore_http_error_codes``: List of HTTP errors to ignore (see :ref:`advanced_topics`) - ``ignore_timeout_errors``: Do not report errors when the timeout is hit - ``ignore_too_many_redirects``: Ignore redirect loops (see :ref:`advanced_topics`) -- ``user_visible_url``: Different URL to show in reports (e.g. when watched URL is a REST API URL, and you want to show a webpage) (Note: ``url`` implies ``kind: url``) @@ -130,6 +131,7 @@ - ``treat_new_as_changed``: Will treat jobs that don't have any historic data as ``CHANGED`` instead of ``NEW`` (and create a diff for new jobs) - ``compared_versions``: Number of versions to compare for similarity - ``kind`` (redundant): Either ``url``, ``shell`` or ``browser``. Automatically derived from the unique key (``url``, ``command`` or ``navigate``) of the job type +- ``user_visible_url``: Different URL to show in reports (e.g. when watched URL is a REST API URL, and you want to show a webpage) Settings keys for all jobs at once diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/docs/source/reporters.rst new/urlwatch-2.24/docs/source/reporters.rst --- old/urlwatch-2.23/docs/source/reporters.rst 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/docs/source/reporters.rst 2021-11-07 09:34:54.000000000 +0100 @@ -106,6 +106,17 @@ chat_id: '88888888' # the chat id where the messages should be sent enabled: true +Messages can be sent silently (``silent``) if you prefer notifications +with no sounds, and monospace formatted (``monospace``). +By default notifications are not silent and no formatting is done. + +.. code:: yaml + + telegram: + # ... + silent: true # message is sent silently + monospace: true # display message as pre-formatted code block + To set up Telegram, from your Telegram app, chat up BotFather (New Message, Search, ???BotFather???), then say ``/newbot`` and follow the instructions. Eventually it will tell you the bot token (in the form @@ -173,7 +184,7 @@ enabled: true embed: true subject: '{count} changes: {jobs}' - + To set up Discord, from your Discord Server settings, select Integration and then create a "New Webhook", give the webhook a name to post under, select a channel, push "Copy Webhook URL" and paste it into the configuration as seen above. Embedded content might be easier to read and identify individual reports. subject preceeds the embedded report and is only used when embed is true. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/__init__.py new/urlwatch-2.24/lib/urlwatch/__init__.py --- old/urlwatch-2.23/lib/urlwatch/__init__.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/__init__.py 2021-11-07 09:34:54.000000000 +0100 @@ -12,5 +12,5 @@ __author__ = 'Thomas Perl <[email protected]>' __license__ = 'BSD' __url__ = 'https://thp.io/2008/urlwatch/' -__version__ = '2.23' +__version__ = '2.24' __user_agent__ = '%s/%s (+https://thp.io/2008/urlwatch/info.html)' % (pkgname, __version__) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/config.py new/urlwatch-2.24/lib/urlwatch/config.py --- old/urlwatch-2.23/lib/urlwatch/config.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/config.py 2021-11-07 09:34:54.000000000 +0100 @@ -75,6 +75,7 @@ parser = argparse.ArgumentParser(description=urlwatch.__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('joblist', metavar='JOB', type=int, nargs="*", help='index of job(s) to run, as numbered according to the --list command. If none specified, then all jobs will be run.') parser.add_argument('--version', action='version', version='%(prog)s {}'.format(urlwatch.__version__)) parser.add_argument('-v', '--verbose', action='store_true', help='show debug output') group = parser.add_argument_group('files and directories') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/filters.py new/urlwatch-2.24/lib/urlwatch/filters.py --- old/urlwatch-2.23/lib/urlwatch/filters.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/filters.py 2021-11-07 09:34:54.000000000 +0100 @@ -39,6 +39,7 @@ import sys import subprocess import io +import csv from enum import Enum from lxml import etree @@ -330,6 +331,53 @@ method=method, options=options) +class Csv2TextFilter(FilterBase): + """Convert CSV to plaintext""" + + __kind__ = 'csv2text' + + __supported_subfilters__ = { + 'format_message': 'A format string with the headers that will be outputted for each csv ' + 'line (header will be lower-cased)', + 'ignore_header': 'If your format string is number based, but the CSV has headers, ' + 'this flag will force ignoring the header.', + 'has_header': 'If specified and true - use the first line as a header. ' + 'If false - force ignore first line as header (treat it as data). ' + 'If not specified csv.Sniffer will be used.', + } + + __default_subfilter__ = 'format_message' + + def filter(self, data, subfilter): + has_header_config = subfilter.get('has_header') + + if has_header_config is None: + has_header = csv.Sniffer().has_header(data) + else: + has_header = has_header_config + + reader = csv.reader(data.split('\n')) + data_list = list(reader) + header = None + + if has_header: + header = data_list.pop(0) + + header = [i.lower() for i in header] + message = subfilter['format_message'] + ignore_header = subfilter['ignore_header'] + + lines = [] + for i in data_list: + if header and not ignore_header: + legend = dict(zip(header, i)) + lines.append(message.format(**legend)) + else: + lines.append(message.format(*i)) + + return '\n'.join(lines) + + class Pdf2TextFilter(FilterBase): """Convert PDF to plaintext""" # Requires data to be in bytes (not unicode) @@ -809,6 +857,31 @@ return separator.join(sorted(data.split(separator), key=str.casefold, reverse=reverse)) +class RemoveDuplicateLinesFilter(FilterBase): + """Remove duplicate lines""" + + __kind__ = 'remove-duplicate-lines' + + __supported_subfilters__ = { + 'separator': 'Item separator (default: newline)', + } + + __default_subfilter__ = 'separator' + + def filter(self, data, subfilter): + separator = subfilter.get('separator', '\n') + data_lines = data.split(separator) + + def get_unique_lines(lines): + seen = set() + for line in lines: + if line not in seen: + yield line + seen.add(line) + + return separator.join(get_unique_lines(data_lines)) + + class ReverseFilter(FilterBase): """Reverse input items""" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/jobs.py new/urlwatch-2.24/lib/urlwatch/jobs.py --- old/urlwatch-2.23/lib/urlwatch/jobs.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/jobs.py 2021-11-07 09:34:54.000000000 +0100 @@ -180,10 +180,11 @@ class Job(JobBase): __required__ = () - __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed') + __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed', 'user_visible_url') # determine if hyperlink "a" tag is used in HtmlReporter - LOCATION_IS_URL = False + def location_is_url(self): + return re.match("^([a-zA-Z0-9+.-]+)://", self.get_location()) def pretty_name(self): return self.name if self.name else self.get_location() @@ -198,7 +199,7 @@ __optional__ = () def get_location(self): - return self.command + return self.user_visible_url or self.command def retrieve(self, job_state): process = subprocess.Popen(self.command, stdout=subprocess.PIPE, shell=True) @@ -221,9 +222,8 @@ __required__ = ('url',) __optional__ = ('cookies', 'data', 'method', 'ssl_no_verify', 'ignore_cached', 'http_proxy', 'https_proxy', 'headers', 'ignore_connection_errors', 'ignore_http_error_codes', 'encoding', 'timeout', - 'ignore_timeout_errors', 'ignore_too_many_redirects', 'user_visible_url') + 'ignore_timeout_errors', 'ignore_too_many_redirects') - LOCATION_IS_URL = True CHARSET_RE = re.compile('text/(html|plain); charset=([^;]*)') def get_location(self): @@ -251,12 +251,14 @@ headers['Cache-Control'] = 'max-age=172800' headers['Expires'] = email.utils.formatdate() - if self.method is None: - self.method = "GET" if self.data is not None: - self.method = "POST" + if self.method is None: + self.method = "POST" headers['Content-type'] = 'application/x-www-form-urlencoded' - logger.info('Sending POST request to %s', self.url) + logger.info('Sending %s request to %s', self.method, self.url) + + if self.method is None: + self.method = "GET" if self.http_proxy is not None: proxies['http'] = self.http_proxy @@ -366,10 +368,8 @@ __optional__ = ('wait_until',) - LOCATION_IS_URL = True - def get_location(self): - return self.navigate + return self.user_visible_url or self.navigate def main_thread_enter(self): from .browser import BrowserContext diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/mailer.py new/urlwatch-2.24/lib/urlwatch/mailer.py --- old/urlwatch-2.23/lib/urlwatch/mailer.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/mailer.py 2021-11-07 09:34:54.000000000 +0100 @@ -95,6 +95,8 @@ if passwd is None: raise ValueError('No password available in keyring for {}, {}' .format(self.smtp_server, self.smtp_user)) + else: + raise ValueError('SMTP auth is enabled, but insecure_password is not set and keyring is not available') s.login(self.smtp_user, passwd) s.sendmail(msg['From'], msg['To'].split(','), msg.as_string()) @@ -106,7 +108,7 @@ self.sendmail_path = sendmail_path def send(self, msg): - p = subprocess.Popen([self.sendmail_path, '-oi', msg['To']], + p = subprocess.Popen([self.sendmail_path, '-oi', '-f', msg['From'], msg['To']], stdin=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/reporters.py new/urlwatch-2.24/lib/urlwatch/reporters.py --- old/urlwatch-2.23/lib/urlwatch/reporters.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/reporters.py 2021-11-07 09:34:54.000000000 +0100 @@ -183,7 +183,7 @@ for job_state in self.report.get_filtered_job_states(self.job_states): job = job_state.job - if job.LOCATION_IS_URL: + if job.location_is_url(): title = '<a href="{location}">{pretty_name}</a>' elif job.pretty_name() != job.get_location(): title = '<span title="{location}">{pretty_name}</span>' @@ -600,11 +600,29 @@ return result + @staticmethod + def _format_body(text: str) -> str: + return "```diff\n{}\n```".format( + text.translate(str.maketrans({'`': r'\`', '\\': r'\\'}))) + def submitToTelegram(self, bot_token, chat_id, text): logger.debug("Sending telegram request to chat id:'{0}'".format(chat_id)) - result = requests.post( - "https://api.telegram.org/bot{0}/sendMessage".format(bot_token), - data={"chat_id": chat_id, "text": text, "disable_web_page_preview": "true"}) + + data = {"chat_id": chat_id, + "text": text, + "disable_notification": self.config.get('silent', False), + "disable_web_page_preview": True} + + if self.config.get('monospace', False): + # all "`" and "\" characters are escaped and text is put inside + # a markdown code block. API docs on formatting messages: + # https://core.telegram.org/bots/api#formatting-options + data.update({ + "text": self._format_body(text), + "parse_mode": "MarkdownV2" + }) + + result = requests.post("https://api.telegram.org/bot{0}/sendMessage".format(bot_token), json=data) try: json_res = result.json() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/storage.py new/urlwatch-2.24/lib/urlwatch/storage.py --- old/urlwatch-2.23/lib/urlwatch/storage.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/storage.py 2021-11-07 09:34:54.000000000 +0100 @@ -126,6 +126,8 @@ 'enabled': False, 'bot_token': '', 'chat_id': '', + 'monospace': False, + 'silent': False, }, 'slack': { 'enabled': False, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/tests/data/filter_documentation_testdata.yaml new/urlwatch-2.24/lib/urlwatch/tests/data/filter_documentation_testdata.yaml --- old/urlwatch-2.23/lib/urlwatch/tests/data/filter_documentation_testdata.yaml 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/tests/data/filter_documentation_testdata.yaml 2021-11-07 09:34:54.000000000 +0100 @@ -368,7 +368,33 @@ "company": "Initech" } ] - output: | + output: |- "Senior Data Scientist" "Python Developer" - "TPS Report Analyst" \ No newline at end of file + "TPS Report Analyst" +https://api.github.com/repos/voxpupuli/puppet-rundeck/tags: + input: | + [ + { + "name": "v6.0.0", + "zipball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/zipball/refs/tags/v6.0.0";, + "tarball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/tarball/refs/tags/v6.0.0";, + "commit": { + "sha": "f539fb125f8fe56cec4f134dd5ef4b5c9be961c5", + "url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/commits/f539fb125f8fe56cec4f134dd5ef4b5c9be961c5"; + }, + "node_id": "MDM6UmVmMTgyMDczMjM6cmVmcy90YWdzL3Y2LjAuMA==" + }, + { + "name": "v5.4.0", + "zipball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/zipball/refs/tags/v5.4.0";, + "tarball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/tarball/refs/tags/v5.4.0";, + "commit": { + "sha": "4689cde2189ac38a36c81275fb13925f5c4a8ca2", + "url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/commits/4689cde2189ac38a36c81275fb13925f5c4a8ca2"; + }, + "node_id": "MDM6UmVmMTgyMDczMjM6cmVmcy90YWdzL3Y1LjQuMA==" + } + ] + output: |- + "v6.0.0" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/tests/test_handler.py new/urlwatch-2.24/lib/urlwatch/tests/test_handler.py --- old/urlwatch-2.23/lib/urlwatch/tests/test_handler.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/tests/test_handler.py 2021-11-07 09:34:54.000000000 +0100 @@ -11,7 +11,7 @@ import os from urlwatch import storage -from urlwatch.config import BaseConfig +from urlwatch.config import CommandConfig from urlwatch.storage import YamlConfigStorage, CacheMiniDBStorage from urlwatch.main import Urlwatch from urlwatch.util import import_module_from_source @@ -87,12 +87,10 @@ assert result.total_errors == 0, "Found #{0} code style errors".format(result.total_errors) -class ConfigForTest(BaseConfig): +class ConfigForTest(CommandConfig): def __init__(self, config, urls, cache, hooks, verbose): (prefix, bindir) = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0]))) - super().__init__('urlwatch', os.path.dirname(__file__), config, urls, cache, hooks, verbose) - self.edit = False - self.edit_hooks = False + super().__init__('urlwatch', os.path.dirname(__file__), bindir, prefix, config, urls, hooks, cache, verbose) @contextlib.contextmanager diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/urlwatch-2.23/lib/urlwatch/worker.py new/urlwatch-2.24/lib/urlwatch/worker.py --- old/urlwatch-2.23/lib/urlwatch/worker.py 2021-04-10 10:48:22.000000000 +0200 +++ new/urlwatch-2.24/lib/urlwatch/worker.py 2021-11-07 09:34:54.000000000 +0100 @@ -51,12 +51,14 @@ def run_jobs(urlwatcher): + if not all(1 <= idx <= len(urlwatcher.jobs) for idx in urlwatcher.urlwatch_config.joblist): + raise ValueError(f'All job indices must be between 1 and {len(urlwatcher.jobs)}: {urlwatcher.urlwatch_config.joblist}') cache_storage = urlwatcher.cache_storage jobs = [job.with_defaults(urlwatcher.config_storage.config) - for job in urlwatcher.jobs] + for (idx, job) in enumerate(urlwatcher.jobs) if ((idx + 1) in urlwatcher.urlwatch_config.joblist or (not urlwatcher.urlwatch_config.joblist))] report = urlwatcher.report - logger.debug('Processing %d jobs', len(jobs)) + logger.debug('Processing %d jobs (out of %d)', len(jobs), len(urlwatcher.jobs)) with contextlib.ExitStack() as exit_stack: for job_state in run_parallel(lambda job_state: job_state.process(), (exit_stack.enter_context(JobState(cache_storage, job)) for job in jobs)):
