Hello community, here is the log from the commit of package python-shodan for openSUSE:Factory checked in at 2018-01-10 23:32:18 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-shodan (Old) and /work/SRC/openSUSE:Factory/.python-shodan.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-shodan" Wed Jan 10 23:32:18 2018 rev:2 rq:557847 version:1.7.7 Changes: -------- --- /work/SRC/openSUSE:Factory/python-shodan/python-shodan.changes 2017-09-05 15:18:51.278584824 +0200 +++ /work/SRC/openSUSE:Factory/.python-shodan.new/python-shodan.changes 2018-01-10 23:32:19.832012325 +0100 @@ -1,0 +2,12 @@ +Sun Dec 10 17:03:19 UTC 2017 - [email protected] + +- update to version 1.7.7 + * Added "shodan data download" command to help download bulk data files + +------------------------------------------------------------------- +Tue Dec 5 08:33:09 UTC 2017 - [email protected] + +- update to version 1.7.6: + * addes "shodan radar" command + +------------------------------------------------------------------- Old: ---- shodan-1.7.5.tar.gz New: ---- shodan-1.7.7.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-shodan.spec ++++++ --- /var/tmp/diff_new_pack.UFqQg6/_old 2018-01-10 23:32:20.427984357 +0100 +++ /var/tmp/diff_new_pack.UFqQg6/_new 2018-01-10 23:32:20.427984357 +0100 @@ -19,7 +19,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} %{!?license: %global license %doc} Name: python-shodan -Version: 1.7.5 +Version: 1.7.7 Release: 0 Summary: Python library and command-line utility for Shodan License: MIT ++++++ shodan-1.7.5.tar.gz -> shodan-1.7.7.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/PKG-INFO new/shodan-1.7.7/PKG-INFO --- old/shodan-1.7.5/PKG-INFO 2017-08-30 18:39:27.000000000 +0200 +++ new/shodan-1.7.7/PKG-INFO 2017-12-08 03:30:21.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: shodan -Version: 1.7.5 +Version: 1.7.7 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/bin/shodan new/shodan-1.7.7/bin/shodan --- old/shodan-1.7.5/bin/shodan 2017-07-17 07:51:28.000000000 +0200 +++ new/shodan-1.7.7/bin/shodan 2017-12-08 03:20:01.000000000 +0100 @@ -9,14 +9,21 @@ The following commands are currently supported: alert + convert count + data download + honeyscore host + info init myip parse + radar scan search + stats + stream """ @@ -302,6 +309,90 @@ click.echo(results['total']) [email protected]() +def data(): + """Bulk data access to Shodan""" + pass + + [email protected](name='list') [email protected]('--dataset', help='See the available files in the given dataset', default=None, type=str) +def data_list(dataset): + """List available datasets or the files within those datasets.""" + # Setup the API connection + key = get_api_key() + api = shodan.Shodan(key) + + if dataset: + # Show the files within this dataset + files = api.data.list_files(dataset) + + for file in files: + click.echo(click.style('{:20s}'.format(file['name']), fg='cyan'), nl=False) + click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) + click.echo('{}'.format(file['url'])) + else: + # If no dataset was provided then show a list of all datasets + datasets = api.data.list_datasets() + + for ds in datasets: + click.echo(click.style('{:15s}'.format(ds['name']), fg='cyan'), nl=False) + click.echo('{}'.format(ds['description'])) + + [email protected](name='download') [email protected]('--chunksize', help='The size of the chunks that are downloaded into memory before writing them to disk.', default=1024, type=int) [email protected]('--filename', '-O', help='Save the file as the provided filename instead of the default.') [email protected]('dataset', metavar='<dataset>') [email protected]('name', metavar='<file>') +def data_download(chunksize, filename, dataset, name): + # Setup the API connection + key = get_api_key() + api = shodan.Shodan(key) + + # Get the file object that the user requested which will contain the URL and total file size + file = None + try: + files = api.data.list_files(dataset) + for tmp in files: + if tmp['name'] == name: + file = tmp + break + except shodan.APIError as e: + raise click.ClickException(e.value) + + # The file isn't available + if not file: + raise click.ClickException('File not found') + + # Start downloading the file + response = requests.get(file['url'], stream=True) + + # Figure out the size of the file based on the headers + filesize = response.headers.get('content-length', None) + if not filesize: + # Fall back to using the filesize provided by the API + filesize = file['size'] + else: + filesize = int(filesize) + + chunk_size = 1024 + limit = filesize / chunk_size + + # Create a default filename based on the dataset and the filename within that dataset + if not filename: + filename = '{}-{}'.format(dataset, name) + + # Open the output file and start writing to it in chunks + with open(filename, 'wb') as fout: + with click.progressbar(response.iter_content(chunk_size=chunk_size), length=limit) as bar: + for chunk in bar: + if chunk: + fout.write(chunk) + + click.echo(click.style('Download completed: {}'.format(filename), 'green')) + + @main.command() @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='<filename>') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/docs/examples/query-summary.rst new/shodan-1.7.7/docs/examples/query-summary.rst --- old/shodan-1.7.5/docs/examples/query-summary.rst 2014-10-05 01:45:40.000000000 +0200 +++ new/shodan-1.7.7/docs/examples/query-summary.rst 2017-12-03 06:42:17.000000000 +0100 @@ -32,18 +32,18 @@ 'port', 'asn', - # We only care about the top 5 countries, this is how we let Shodan know to return 5 instead of the - # default 10 for a facet. If you want to see more than 10, you could do ('country', 1000) for example + # We only care about the top 3 countries, this is how we let Shodan know to return 3 instead of the + # default 5 for a facet. If you want to see more than 5, you could do ('country', 1000) for example # to see the top 1,000 countries for a search query. - ('country', 5), + ('country', 3), ] FACET_TITLES = { - 'org': 'Top 10 Organizations', - 'domain': 'Top 10 Domains', - 'port': 'Top 10 Ports', - 'asn': 'Top 10 Autonomous Systems', - 'country': 'Top 5 Countries', + 'org': 'Top 5 Organizations', + 'domain': 'Top 5 Domains', + 'port': 'Top 5 Ports', + 'asn': 'Top 5 Autonomous Systems', + 'country': 'Top 3 Countries', } # Input validation @@ -89,38 +89,36 @@ Query: apache Total Results: 34612043 - Top 10 Organizations + Top 5 Organizations Amazon.com: 808061 Ecommerce Corporation: 788704 Verio Web Hosting: 760112 Unified Layer: 627827 GoDaddy.com, LLC: 567004 - Top 10 Domains + Top 5 Domains secureserver.net: 562047 unifiedlayer.com: 494399 t-ipconnect.de: 385792 netart.pl: 194817 wanadoo.fr: 151925 - Top 10 Ports + Top 5 Ports 80: 24118703 443: 8330932 8080: 1479050 81: 359025 8443: 231441 - Top 10 Autonomous Systems + Top 5 Autonomous Systems as32392: 580002 as2914: 465786 as26496: 414998 as48030: 332000 as8560: 255774 - Top 5 Countries + Top 3 Countries US: 13227366 DE: 2900530 JP: 2014506 - CN: 1722048 - GB: 1209938 """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/setup.cfg new/shodan-1.7.7/setup.cfg --- old/shodan-1.7.5/setup.cfg 2017-08-30 18:39:27.000000000 +0200 +++ new/shodan-1.7.7/setup.cfg 2017-12-08 03:30:21.000000000 +0100 @@ -1,4 +1,5 @@ [egg_info] tag_build = tag_date = 0 +tag_svn_revision = 0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/setup.py new/shodan-1.7.7/setup.py --- old/shodan-1.7.5/setup.py 2017-08-30 18:38:36.000000000 +0200 +++ new/shodan-1.7.7/setup.py 2017-12-08 03:26:34.000000000 +0100 @@ -6,7 +6,7 @@ setup( name = 'shodan', - version = '1.7.5', + version = '1.7.7', description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', author = 'John Matherly', author_email = '[email protected]', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/shodan/client.py new/shodan-1.7.7/shodan/client.py --- old/shodan-1.7.5/shodan/client.py 2017-07-06 17:42:42.000000000 +0200 +++ new/shodan-1.7.7/shodan/client.py 2017-12-05 06:44:11.000000000 +0100 @@ -44,6 +44,25 @@ :ivar stream: An instance of `shodan.Shodan.Stream` that provides access to the Streaming API. """ + class Data: + + def __init__(self, parent): + self.parent = parent + + def list_datasets(self): + """Returns a list of datasets that the user has permission to download. + + :returns: A list of objects where every object describes a dataset + """ + return self.parent._request('/shodan/data', {}) + + def list_files(self, dataset): + """Returns a list of files that belong to the given dataset. + + :returns: A list of objects where each object contains a 'name', 'size', 'timestamp' and 'url' + """ + return self.parent._request('/shodan/data/{}'.format(dataset), {}) + class Tools: def __init__(self, parent): @@ -125,6 +144,7 @@ self.api_key = key self.base_url = 'https://api.shodan.io' self.base_exploits_url = 'https://exploits.shodan.io' + self.data = self.Data(self) self.exploits = self.Exploits(self) self.labs = self.Labs(self) self.tools = self.Tools(self) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/shodan/stream.py new/shodan-1.7.7/shodan/stream.py --- old/shodan-1.7.5/shodan/stream.py 2017-08-30 18:27:22.000000000 +0200 +++ new/shodan-1.7.7/shodan/stream.py 2017-12-03 23:57:41.000000000 +0100 @@ -46,13 +46,25 @@ req.encoding = 'utf-8' return req - def _iter_stream(self, stream, raw): + def _iter_stream(self, stream, raw, timeout=None): for line in stream.iter_lines(decode_unicode=True): + # The Streaming API sends out heartbeat messages that are newlines + # We want to ignore those messages since they don't contain any data if line: if raw: yield line else: yield json.loads(line) + else: + # If the user specified a timeout then we want to keep track of how long we've + # been getting heartbeat messages and exit the loop if it's been too long since + # we've seen any activity. + if timeout: + # TODO: This is a placeholder for now but since the Streaming API added heartbeats it broke + # the ability to use inactivity timeouts (the connection timeout still works). The timeout is + # mostly needed when doing on-demand scans and wanting to temporarily consume data from a + # network alert. + pass def alert(self, aid=None, timeout=None, raw=False): if aid: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/shodan.egg-info/PKG-INFO new/shodan-1.7.7/shodan.egg-info/PKG-INFO --- old/shodan-1.7.5/shodan.egg-info/PKG-INFO 2017-08-30 18:39:27.000000000 +0200 +++ new/shodan-1.7.7/shodan.egg-info/PKG-INFO 2017-12-08 03:30:20.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: shodan -Version: 1.7.5 +Version: 1.7.7 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.7.5/shodan.egg-info/requires.txt new/shodan-1.7.7/shodan.egg-info/requires.txt --- old/shodan-1.7.5/shodan.egg-info/requires.txt 2017-08-30 18:39:27.000000000 +0200 +++ new/shodan-1.7.7/shodan.egg-info/requires.txt 2017-12-08 03:30:20.000000000 +0100 @@ -1,5 +1,5 @@ -XlsxWriter click click-plugins colorama requests>=2.2.1 +XlsxWriter
