Hello community, here is the log from the commit of package python-shodan for openSUSE:Factory checked in at 2019-09-11 10:35:33 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-shodan (Old) and /work/SRC/openSUSE:Factory/.python-shodan.new.7948 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-shodan" Wed Sep 11 10:35:33 2019 rev:14 rq:729778 version:1.15.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-shodan/python-shodan.changes 2019-07-29 17:30:15.042216492 +0200 +++ /work/SRC/openSUSE:Factory/.python-shodan.new.7948/python-shodan.changes 2019-09-11 10:35:35.283292834 +0200 @@ -1,0 +2,6 @@ +Tue Sep 10 10:22:55 UTC 2019 - Tomáš Chvátal <tchva...@suse.com> + +- Update to 1.15.0: + * New option --skip for the shodan download command to skip results + +------------------------------------------------------------------- Old: ---- shodan-1.14.0.tar.gz New: ---- shodan-1.15.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-shodan.spec ++++++ --- /var/tmp/diff_new_pack.AR8Rfa/_old 2019-09-11 10:35:36.071292606 +0200 +++ /var/tmp/diff_new_pack.AR8Rfa/_new 2019-09-11 10:35:36.075292604 +0200 @@ -19,7 +19,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} %{!?license: %global license %doc} Name: python-shodan -Version: 1.14.0 +Version: 1.15.0 Release: 0 Summary: Python library and command-line utility for Shodan License: MIT ++++++ shodan-1.14.0.tar.gz -> shodan-1.15.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.14.0/CHANGELOG.md new/shodan-1.15.0/CHANGELOG.md --- old/shodan-1.14.0/CHANGELOG.md 2019-07-20 03:14:01.000000000 +0200 +++ new/shodan-1.15.0/CHANGELOG.md 2019-08-13 03:15:35.000000000 +0200 @@ -1,8 +1,12 @@ CHANGELOG ========= +1.15.0 +------ +* New option "--skip" for download command to help users resume a download + 1.14.0 ----------- +------ * New command **shodan version** (#104). * Only change api_key file permissions if needed (#103) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.14.0/PKG-INFO new/shodan-1.15.0/PKG-INFO --- old/shodan-1.14.0/PKG-INFO 2019-07-20 03:16:22.000000000 +0200 +++ new/shodan-1.15.0/PKG-INFO 2019-08-13 03:40:03.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: shodan -Version: 1.14.0 +Version: 1.15.0 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.14.0/setup.py new/shodan-1.15.0/setup.py --- old/shodan-1.14.0/setup.py 2019-07-20 03:14:06.000000000 +0200 +++ new/shodan-1.15.0/setup.py 2019-08-13 03:14:12.000000000 +0200 @@ -7,7 +7,7 @@ setup( name='shodan', - version='1.14.0', + version='1.15.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.14.0/shodan/__main__.py new/shodan-1.15.0/shodan/__main__.py --- old/shodan-1.14.0/shodan/__main__.py 2019-07-20 03:09:37.000000000 +0200 +++ new/shodan-1.15.0/shodan/__main__.py 2019-08-13 03:22:56.000000000 +0200 @@ -205,9 +205,10 @@ @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) +@click.option('--skip', help='The number of results to skip when starting the download.', default=0, type=int) @click.argument('filename', metavar='<filename>') @click.argument('query', metavar='<search query>', nargs=-1) -def download(limit, filename, query): +def download(limit, skip, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() @@ -247,11 +248,15 @@ # A limit of -1 means that we should download all the data if limit <= 0: limit = total + + # Adjust the total number of results we should expect to download if the user is skipping results + if skip > 0: + limit -= skip with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False) + cursor = api.search_cursor(query, minify=False, skip=skip) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.14.0/shodan/client.py new/shodan-1.15.0/shodan/client.py --- old/shodan-1.14.0/shodan/client.py 2019-07-20 03:09:37.000000000 +0200 +++ new/shodan-1.15.0/shodan/client.py 2019-08-13 03:22:34.000000000 +0200 @@ -430,7 +430,7 @@ return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, skip=0): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -444,16 +444,27 @@ :type minify: bool :param retries: (optional) How often to retry the search in case it times out :type retries: int + :param skip: (optional) Number of results to skip + :type skip: int :returns: A search cursor that can be used as an iterator/ generator. """ page = 1 tries = 0 + + # Placeholder results object to make the while loop below easier results = { - 'matches': [], + 'matches': [True], 'total': None, } - while page == 1 or results['matches']: + + # Convert the number of skipped records into a page number + if skip > 0: + # Each page returns 100 results so find the nearest page that we want + # the cursor to skip to. + page += int(skip / 100) + + while results['matches']: try: results = self.search(query, minify=minify, page=page) for banner in results['matches']: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.14.0/shodan.egg-info/PKG-INFO new/shodan-1.15.0/shodan.egg-info/PKG-INFO --- old/shodan-1.14.0/shodan.egg-info/PKG-INFO 2019-07-20 03:16:22.000000000 +0200 +++ new/shodan-1.15.0/shodan.egg-info/PKG-INFO 2019-08-13 03:40:03.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: shodan -Version: 1.14.0 +Version: 1.15.0 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly