Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-shodan for openSUSE:Factory checked in at 2021-01-26 14:47:16 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-shodan (Old) and /work/SRC/openSUSE:Factory/.python-shodan.new.28504 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-shodan" Tue Jan 26 14:47:16 2021 rev:26 rq:866748 version:1.25.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-shodan/python-shodan.changes 2020-11-09 13:59:02.831750052 +0100 +++ /work/SRC/openSUSE:Factory/.python-shodan.new.28504/python-shodan.changes 2021-01-26 14:51:07.363789895 +0100 @@ -1,0 +2,6 @@ +Tue Jan 26 09:02:37 UTC 2021 - Sebastian Wagner <[email protected]> + +- Update to version 1.25.0: + - Add new CLI command: shodan alert download [--alert-id=] <filename> + +------------------------------------------------------------------- Old: ---- shodan-1.24.0.tar.gz New: ---- shodan-1.25.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-shodan.spec ++++++ --- /var/tmp/diff_new_pack.Z99PIi/_old 2021-01-26 14:51:08.103790903 +0100 +++ /var/tmp/diff_new_pack.Z99PIi/_new 2021-01-26 14:51:08.103790903 +0100 @@ -1,7 +1,7 @@ # # spec file for package python-shodan # -# Copyright (c) 2020 SUSE LLC +# Copyright (c) 2021 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -19,7 +19,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} %{!?license: %global license %doc} Name: python-shodan -Version: 1.24.0 +Version: 1.25.0 Release: 0 Summary: Python library and command-line utility for Shodan License: MIT ++++++ shodan-1.24.0.tar.gz -> shodan-1.25.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/CHANGELOG.md new/shodan-1.25.0/CHANGELOG.md --- old/shodan-1.24.0/CHANGELOG.md 2020-04-06 22:00:31.000000000 +0200 +++ new/shodan-1.25.0/CHANGELOG.md 2021-01-25 23:31:44.000000000 +0100 @@ -1,6 +1,14 @@ CHANGELOG ========= +1.25.0 +------ +* Add new CLI command: shodan alert download + +1.24.0 +------ +* Add new CLI command: shodan alert stats + 1.23.0 ------ * Add new CLI command: shodan alert domain diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/PKG-INFO new/shodan-1.25.0/PKG-INFO --- old/shodan-1.24.0/PKG-INFO 2020-10-12 00:55:18.000000000 +0200 +++ new/shodan-1.25.0/PKG-INFO 2021-01-25 23:35:15.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: shodan -Version: 1.24.0 +Version: 1.25.0 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/setup.py new/shodan-1.25.0/setup.py --- old/shodan-1.24.0/setup.py 2020-10-12 00:24:01.000000000 +0200 +++ new/shodan-1.25.0/setup.py 2021-01-25 23:30:39.000000000 +0100 @@ -9,7 +9,7 @@ setup( name='shodan', - version='1.24.0', + version='1.25.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/shodan/__main__.py new/shodan-1.25.0/shodan/__main__.py --- old/shodan-1.24.0/shodan/__main__.py 2020-09-08 02:37:18.000000000 +0200 +++ new/shodan-1.25.0/shodan/__main__.py 2020-11-02 20:46:46.000000000 +0100 @@ -256,10 +256,9 @@ @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) [email protected]('--skip', help='The number of results to skip when starting the download.', default=0, type=int) @click.argument('filename', metavar='<filename>') @click.argument('query', metavar='<search query>', nargs=-1) -def download(limit, skip, filename, query): +def download(limit, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() @@ -299,15 +298,11 @@ # A limit of -1 means that we should download all the data if limit <= 0: limit = total - - # Adjust the total number of results we should expect to download if the user is skipping results - if skip > 0: - limit -= skip with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False, skip=skip) + cursor = api.search_cursor(query, minify=False) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/shodan/cli/alert.py new/shodan-1.25.0/shodan/cli/alert.py --- old/shodan-1.24.0/shodan/cli/alert.py 2020-10-12 00:48:51.000000000 +0200 +++ new/shodan-1.25.0/shodan/cli/alert.py 2021-01-25 23:30:07.000000000 +0100 @@ -4,7 +4,10 @@ from collections import defaultdict from operator import itemgetter +from shodan import APIError from shodan.cli.helpers import get_api_key +from shodan.helpers import open_file, write_banner +from time import sleep MAX_QUERY_LENGTH = 1000 @@ -138,6 +141,86 @@ click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') [email protected](name='download') [email protected]('filename', metavar='<filename>', type=str) [email protected]('--alert-id', help='Specific alert ID to download the data of', default=None) +def alert_download(filename, alert_id): + """Download all information for monitored networks/ IPs.""" + key = get_api_key() + + api = shodan.Shodan(key) + ips = set() + networks = set() + + # Helper method to process batches of IPs + def batch(iterable, size=1): + iter_length = len(iterable) + for ndx in range(0, iter_length, size): + yield iterable[ndx:min(ndx + size, iter_length)] + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + if alert_id: + alerts = [api.alerts(aid=alert_id.strip())] + else: + alerts = api.alerts() + + click.echo('Compiling list of networks/ IPs to download...') + for alert in alerts: + for net in alert['filters']['ip']: + if '/' in net: + networks.add(net) + else: + ips.add(net) + + click.echo('Downloading...') + with open_file(filename) as fout: + # Check if the user is able to use batch IP lookups + batch_size = 1 + if len(ips) > 0: + api_info = api.info() + if api_info['plan'] in ['corp', 'stream-100']: + batch_size = 100 + + # Convert it to a list so we can index into it + ips = list(ips) + + # Grab all the IP information + for ip in batch(ips, size=batch_size): + try: + click.echo(ip) + results = api.host(ip) + if not isinstance(results, list): + results = [results] + + for host in results: + for banner in host['data']: + write_banner(fout, banner) + except APIError: + pass + sleep(1) # Slow down a bit to make sure we don't hit the rate limit + + # Grab all the network ranges + for net in networks: + try: + counter = 0 + click.echo(net) + for banner in api.search_cursor('net:{}'.format(net)): + write_banner(fout, banner) + + # Slow down a bit to make sure we don't hit the rate limit + if counter % 100 == 0: + sleep(1) + counter += 1 + except APIError: + pass + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') + + @alert.command(name='info') @click.argument('alert', metavar='<alert id>') def alert_info(alert): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/shodan/client.py new/shodan-1.25.0/shodan/client.py --- old/shodan-1.24.0/shodan/client.py 2020-01-07 06:36:59.000000000 +0100 +++ new/shodan-1.25.0/shodan/client.py 2020-11-02 20:47:47.000000000 +0100 @@ -504,7 +504,7 @@ return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5, skip=0): + def search_cursor(self, query, minify=True, retries=5): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -518,8 +518,6 @@ :type minify: bool :param retries: (optional) How often to retry the search in case it times out :type retries: int - :param skip: (optional) Number of results to skip - :type skip: int :returns: A search cursor that can be used as an iterator/ generator. """ @@ -532,12 +530,6 @@ 'total': None, } - # Convert the number of skipped records into a page number - if skip > 0: - # Each page returns 100 results so find the nearest page that we want - # the cursor to skip to. - page += int(skip / 100) - while results['matches']: try: results = self.search(query, minify=minify, page=page) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.24.0/shodan.egg-info/PKG-INFO new/shodan-1.25.0/shodan.egg-info/PKG-INFO --- old/shodan-1.24.0/shodan.egg-info/PKG-INFO 2020-10-12 00:55:18.000000000 +0200 +++ new/shodan-1.25.0/shodan.egg-info/PKG-INFO 2021-01-25 23:35:15.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: shodan -Version: 1.24.0 +Version: 1.25.0 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly
