This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository imposm-parser.
commit dc2c4a49646f923ef2b6b2d1180f6a96b8132cc0 Author: David Paleino <da...@debian.org> Date: Mon Nov 4 17:24:25 2013 +0100 Imported Upstream version 1.0.5 --- CHANGES | 12 +++++++ MANIFEST.in | 2 +- PKG-INFO | 26 ++++++++++----- README => README.rst | 8 ++--- imposm.parser.egg-info/PKG-INFO | 26 ++++++++++----- imposm.parser.egg-info/SOURCES.txt | 3 +- imposm/parser/example.py | 45 -------------------------- imposm/parser/pbf/multiproc.py | 33 +++++++++---------- imposm/parser/pbf/parser.py | 2 +- imposm/parser/xml/multiproc.py | 65 ++++++++++++++++++++------------------ setup.py | 8 ++--- 11 files changed, 110 insertions(+), 120 deletions(-) diff --git a/CHANGES b/CHANGES index 579d1e5..0c5b554 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,18 @@ Changelog --------- +1.0.5 2013-09-13 +~~~~~~~~~~~~~~~~ + +- support PBF without granularity value +- improved support for non-pretty-printed XML + +1.0.4 2012-12-10 +~~~~~~~~~~~~~~~~ + +- improved support for non-pretty-printed XML +- fixed dependency check for multiprocessing + 1.0.3 2011-07-21 ~~~~~~~~~~~~~~~~ diff --git a/MANIFEST.in b/MANIFEST.in index eba65a9..7c6f728 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include MANIFEST.in -include README +include README.rst include CHANGES include LICENSE include setup.py diff --git a/PKG-INFO b/PKG-INFO index e89360c..3d16631 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,14 +1,12 @@ Metadata-Version: 1.0 Name: imposm.parser -Version: 1.0.3 +Version: 1.0.5 Summary: Fast and easy OpenStreetMap XML/PBF parser. -Home-page: http://dev.omniscale.net/imposm.parser/ +Home-page: http://imposm.org/docs/imposm.parser/latest/ Author: Oliver Tonnhofer Author-email: o...@omniscale.de License: Apache Software License 2.0 -Description: .. # -*- restructuredtext -*- - - imposm.parser - OpenStreetMap XML/PBF parser for Python +Description: imposm.parser - OpenStreetMap XML/PBF parser for Python ======================================================= ``imposm.parser`` is a Python library that parses OpenStreetMap data in `XML <http://wiki.openstreetmap.org/wiki/.osm>`_ and `PBF <http://wiki.openstreetmap.org/wiki/PBF_Format>`_ format. @@ -28,7 +26,7 @@ Description: .. # -*- restructuredtext -*- # simple class that handles the parsed OSM data. class HighwayCounter(object): highways = 0 - + def ways(self, ways): # callback method for ways for osmid, tags, refs in ways: @@ -39,7 +37,7 @@ Description: .. # -*- restructuredtext -*- counter = HighwayCounter() p = OSMParser(concurrency=4, ways_callback=counter.ways) p.parse('germany.osm.pbf') - + # done print counter.highways @@ -47,11 +45,23 @@ Description: .. # -*- restructuredtext -*- Source and issue tracker ------------------------ - Source code and issue tracker are available at `<https://bitbucket.org/olt/imposm.parser/src>`_. + Source code and issue tracker are available at `<https://github.com/omniscale/imposm-parser>`_. Changelog --------- + 1.0.5 2013-09-13 + ~~~~~~~~~~~~~~~~ + + - support PBF without granularity value + - improved support for non-pretty-printed XML + + 1.0.4 2012-12-10 + ~~~~~~~~~~~~~~~~ + + - improved support for non-pretty-printed XML + - fixed dependency check for multiprocessing + 1.0.3 2011-07-21 ~~~~~~~~~~~~~~~~ diff --git a/README b/README.rst similarity index 90% rename from README rename to README.rst index c03ad8c..48322e8 100644 --- a/README +++ b/README.rst @@ -1,5 +1,3 @@ -.. # -*- restructuredtext -*- - imposm.parser - OpenStreetMap XML/PBF parser for Python ======================================================= @@ -20,7 +18,7 @@ Here is an example that parses an OSM file and counts all ways that are tagged a # simple class that handles the parsed OSM data. class HighwayCounter(object): highways = 0 - + def ways(self, ways): # callback method for ways for osmid, tags, refs in ways: @@ -31,7 +29,7 @@ Here is an example that parses an OSM file and counts all ways that are tagged a counter = HighwayCounter() p = OSMParser(concurrency=4, ways_callback=counter.ways) p.parse('germany.osm.pbf') - + # done print counter.highways @@ -39,5 +37,5 @@ Here is an example that parses an OSM file and counts all ways that are tagged a Source and issue tracker ------------------------ -Source code and issue tracker are available at `<https://bitbucket.org/olt/imposm.parser/src>`_. +Source code and issue tracker are available at `<https://github.com/omniscale/imposm-parser>`_. diff --git a/imposm.parser.egg-info/PKG-INFO b/imposm.parser.egg-info/PKG-INFO index e89360c..3d16631 100644 --- a/imposm.parser.egg-info/PKG-INFO +++ b/imposm.parser.egg-info/PKG-INFO @@ -1,14 +1,12 @@ Metadata-Version: 1.0 Name: imposm.parser -Version: 1.0.3 +Version: 1.0.5 Summary: Fast and easy OpenStreetMap XML/PBF parser. -Home-page: http://dev.omniscale.net/imposm.parser/ +Home-page: http://imposm.org/docs/imposm.parser/latest/ Author: Oliver Tonnhofer Author-email: o...@omniscale.de License: Apache Software License 2.0 -Description: .. # -*- restructuredtext -*- - - imposm.parser - OpenStreetMap XML/PBF parser for Python +Description: imposm.parser - OpenStreetMap XML/PBF parser for Python ======================================================= ``imposm.parser`` is a Python library that parses OpenStreetMap data in `XML <http://wiki.openstreetmap.org/wiki/.osm>`_ and `PBF <http://wiki.openstreetmap.org/wiki/PBF_Format>`_ format. @@ -28,7 +26,7 @@ Description: .. # -*- restructuredtext -*- # simple class that handles the parsed OSM data. class HighwayCounter(object): highways = 0 - + def ways(self, ways): # callback method for ways for osmid, tags, refs in ways: @@ -39,7 +37,7 @@ Description: .. # -*- restructuredtext -*- counter = HighwayCounter() p = OSMParser(concurrency=4, ways_callback=counter.ways) p.parse('germany.osm.pbf') - + # done print counter.highways @@ -47,11 +45,23 @@ Description: .. # -*- restructuredtext -*- Source and issue tracker ------------------------ - Source code and issue tracker are available at `<https://bitbucket.org/olt/imposm.parser/src>`_. + Source code and issue tracker are available at `<https://github.com/omniscale/imposm-parser>`_. Changelog --------- + 1.0.5 2013-09-13 + ~~~~~~~~~~~~~~~~ + + - support PBF without granularity value + - improved support for non-pretty-printed XML + + 1.0.4 2012-12-10 + ~~~~~~~~~~~~~~~~ + + - improved support for non-pretty-printed XML + - fixed dependency check for multiprocessing + 1.0.3 2011-07-21 ~~~~~~~~~~~~~~~~ diff --git a/imposm.parser.egg-info/SOURCES.txt b/imposm.parser.egg-info/SOURCES.txt index d31e249..76b071f 100644 --- a/imposm.parser.egg-info/SOURCES.txt +++ b/imposm.parser.egg-info/SOURCES.txt @@ -1,7 +1,7 @@ CHANGES LICENSE MANIFEST.in -README +README.rst osm.proto setup.py imposm/__init__.py @@ -11,7 +11,6 @@ imposm.parser.egg-info/dependency_links.txt imposm.parser.egg-info/namespace_packages.txt imposm.parser.egg-info/top_level.txt imposm/parser/__init__.py -imposm/parser/example.py imposm/parser/simple.py imposm/parser/util.py imposm/parser/pbf/__init__.py diff --git a/imposm/parser/example.py b/imposm/parser/example.py deleted file mode 100644 index 59ae9f0..0000000 --- a/imposm/parser/example.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2011 Omniscale GmbH & Co. KG -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import multiprocessing -from imposm.parser import OSMParser - -if __name__ == '__main__': - - class Counter(object): - def __init__(self): - self.coords_counter = 0 - self.nodes_counter = 0 - self.ways_counter = 0 - self.relations_counter = 0 - - def incr_coords(self, coords): - self.coords_counter += len(coords) - def incr_nodes(self, nodes): - self.nodes_counter += len(nodes) - def incr_ways(self, ways): - self.ways_counter += len(ways) - def incr_relations(self, relations): - self.relations_counter += len(relations) - - counter = Counter() - - p = OSMParser( - nodes_callback=counter.incr_nodes, - coords_callback=counter.incr_coords, - ways_callback=counter.incr_ways, - relations_callback=counter.incr_relations) - p.parse(sys.argv[1]) - print counter.coords_counter, counter.nodes_counter, counter.ways_counter, counter.relations_counter \ No newline at end of file diff --git a/imposm/parser/pbf/multiproc.py b/imposm/parser/pbf/multiproc.py index c5be9dc..b5a0048 100644 --- a/imposm/parser/pbf/multiproc.py +++ b/imposm/parser/pbf/multiproc.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale GmbH & Co. KG -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,10 +19,11 @@ from imposm.parser.util import setproctitle class PBFParserProcess(PBFParser, multiprocessing.Process): def __init__(self, pos_queue, *args, **kw): - self.pos_queue = pos_queue - PBFParser.__init__(self, *args, **kw) multiprocessing.Process.__init__(self) - + PBFParser.__init__(self, *args, **kw) + self.daemon = True + self.pos_queue = pos_queue + def run(self): setproctitle('imposm pbf parser') while True: @@ -30,7 +31,7 @@ class PBFParserProcess(PBFParser, multiprocessing.Process): if pos is None: self.pos_queue.task_done() break - + self.parse(pos['filename'], offset=pos['blob_pos'], size=pos['blob_size']) self.pos_queue.task_done() @@ -39,7 +40,7 @@ class PBFMultiProcParser(object): nodes_tag_filter = None ways_tag_filter = None relations_tag_filter = None - + def __init__(self, pool_size, nodes_queue=None, ways_queue=None, relations_queue=None, coords_queue=None, marshal_elem_data=False): self.pool_size = pool_size @@ -62,12 +63,12 @@ class PBFMultiProcParser(object): ) pool.append(proc) proc.start() - + reader = PBFFile(filename) - + for pos in reader.blob_offsets(): pos_queue.put(pos) - + pos_queue.join() for proc in pool: @@ -90,12 +91,12 @@ if __name__ == '__main__': queue.task_done() print type, count return count - - + + nodes_queue = multiprocessing.JoinableQueue(128) ways_queue = multiprocessing.JoinableQueue(128) relations_queue = multiprocessing.JoinableQueue(128) - + procs = [ multiprocessing.Process(target=count_proc('nodes', nodes_queue)), multiprocessing.Process(target=count_proc('ways', ways_queue)), @@ -103,11 +104,11 @@ if __name__ == '__main__': ] for proc in procs: proc.start() - + parser = PBFMultiProcParser(2, nodes_queue=nodes_queue, ways_queue=ways_queue, relations_queue=relations_queue) parser.parse(sys.argv[1]) - + nodes_queue.put(None) ways_queue.put(None) relations_queue.put(None) diff --git a/imposm/parser/pbf/parser.py b/imposm/parser/pbf/parser.py index b1adf1f..d4f8d01 100644 --- a/imposm/parser/pbf/parser.py +++ b/imposm/parser/pbf/parser.py @@ -192,7 +192,7 @@ class PrimitiveBlockParser(object): for group in self.primitivegroup: dense = group.dense if dense: - granularity = self.primitive_block.granularity + granularity = self.primitive_block.granularity or 100 lat_offset = self.primitive_block.lat_offset or 0 lon_offset = self.primitive_block.lon_offset or 0 coord_scale = 0.000000001 diff --git a/imposm/parser/xml/multiproc.py b/imposm/parser/xml/multiproc.py index e6fb6e8..ab6b92a 100644 --- a/imposm/parser/xml/multiproc.py +++ b/imposm/parser/xml/multiproc.py @@ -1,11 +1,11 @@ # Copyright 2011 Omniscale GmbH & Co. KG -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,14 +32,14 @@ class MMapReader(object): self.m = m self.m.seek(0) self.size = size - + def read(self, size=None): if size is None: size = self.size - self.m.tell() else: size = min(self.size - self.m.tell(), size) return self.m.read(size) - + def readline(self): cur_pos = self.m.tell() if cur_pos >= self.size: @@ -47,17 +47,17 @@ class MMapReader(object): nl_pos = self.m.find('\n') self.m.seek(cur_pos) return self.m.read(nl_pos-cur_pos) - + def seek(self, n): self.m.seek(n) class XMLParserProcess(XMLParser, multiprocessing.Process): def __init__(self, mmap_pool, mmap_queue, *args, **kw): - self.mmap_pool = mmap_pool - self.mmap_queue = mmap_queue - XMLParser.__init__(self, *args, **kw) multiprocessing.Process.__init__(self) + XMLParser.__init__(self, *args, **kw) self.daemon = True + self.mmap_pool = mmap_pool + self.mmap_queue = mmap_queue def run(self): setproctitle('imposm xml parser') @@ -77,7 +77,7 @@ class XMLMultiProcParser(object): nodes_tag_filter = None ways_tag_filter = None relations_tag_filter = None - + def __init__(self, pool_size, nodes_queue=None, ways_queue=None, relations_queue=None, coords_queue=None, marshal_elem_data=False): self.pool_size = pool_size @@ -90,10 +90,10 @@ class XMLMultiProcParser(object): self.mmap_pool = MMapPool(pool_size*8, xml_chunk_size*8) self.mmap_queue = multiprocessing.JoinableQueue(8) self.marshal_elem_data = marshal_elem_data - + def parse(self, stream): assert not self.pool - + for _ in xrange(self.pool_size): proc = XMLParserProcess(self.mmap_pool, self.mmap_queue, nodes_callback=self.nodes_callback, coords_callback=self.coords_callback, ways_callback=self.ways_callback, @@ -105,16 +105,16 @@ class XMLMultiProcParser(object): ) self.pool.append(proc) proc.start() - + chunker = XMLChunker(stream, self.mmap_pool, xml_chunk_size=READ_SIZE) chunker.read(self.mmap_queue, coords_callback=self.coords_callback) - + self.mmap_queue.join() for proc in self.pool: self.mmap_queue.put((None, None)) for proc in self.pool: proc.join() - + class MMapPool(object): """ @@ -127,11 +127,11 @@ class MMapPool(object): self.pool = [mmap.mmap(-1, mmap_size) for _ in range(n)] self.free_mmaps = set(range(n)) self.free_queue = multiprocessing.JoinableQueue() - + def new(self): """ Return a free mmap file. - + :returns: index, mmap file """ if not self.free_mmaps: @@ -146,7 +146,7 @@ class MMapPool(object): break mmap_idx = self.free_mmaps.pop() return mmap_idx, self.pool[mmap_idx] - + def join(self): while len(self.free_mmaps) < self.n: self.free_mmaps.add(self.free_queue.get()) @@ -167,10 +167,10 @@ class MMapPool(object): class XMLChunker(object): """ Reads and chunks OSM XML file. - + Reads OSM XML from `stream` and writes chunks of it into mmap files from the `mmap_pool`. - + :params xml_chunk_size: chunk XML after this many bytes """ def __init__(self, stream, mmap_pool, xml_chunk_size): @@ -200,13 +200,14 @@ class XMLChunker(object): def read(self, mmaps_queue, coords_callback=None): """ - Read and chunk all + Read and chunk all """ coord_node_match = None xml_nodes = self._new_xml_outstream() coords = [] coord_node_re_match = re.compile(r'^\s*<node id="(\d+)" .*lat="([-0-9.]+)" ' 'lon="([-0-9.]+)".*/>').match + node_re_match = re.compile(r'^\s*<node .*/>').match xml_nodes.write(self._last_line) split = False line = '' @@ -224,9 +225,9 @@ class XMLChunker(object): else: xml_nodes.write(line) if split: - if (line.lstrip().startswith('</') + if (line.rstrip().endswith(('</way>', '</node>', '</relation>')) or (coords_callback and coord_node_match) - or (not coords_callback and coord_node_re_match(line))): + or (not coords_callback and node_re_match(line))): mmaps_queue.put(self._finished_xml_outstream(line, xml_nodes)) xml_nodes = self._new_xml_outstream() split = False @@ -234,7 +235,11 @@ class XMLChunker(object): split = True if coords_callback: coords_callback(coords) - + + # we are at the end of the stream and assume we wrote the end tag + # to xml_nodes. we set line to closing tag here to avoid additional + # end tag in case the last line(s) is blank + line = '</osm' mmaps_queue.put(self._finished_xml_outstream(line, xml_nodes)) if __name__ == '__main__': @@ -253,13 +258,13 @@ if __name__ == '__main__': queue.task_done() print type, count return count - - + + nodes_queue = multiprocessing.JoinableQueue(128) ways_queue = multiprocessing.JoinableQueue(128) relations_queue = multiprocessing.JoinableQueue(128) - + procs = [ multiprocessing.Process(target=count_proc('nodes', nodes_queue)), multiprocessing.Process(target=count_proc('ways', ways_queue)), @@ -267,14 +272,14 @@ if __name__ == '__main__': ] for proc in procs: proc.start() - + parser = XMLMultiProcParser(open(sys.argv[1]), 2, nodes_queue=nodes_queue, ways_queue=ways_queue, relations_queue=relations_queue) parser.start() - + nodes_queue.put(None) nodes_queue.join() - + ways_queue.put(None) ways_queue.join() relations_queue.put(None) diff --git a/setup.py b/setup.py index 9160157..1af1a03 100644 --- a/setup.py +++ b/setup.py @@ -31,17 +31,17 @@ class build_ext_with_protpbuf(build_ext): install_requires = [] -if platform.python_version_tuple() < ('2', '6'): +if tuple(map(str, platform.python_version_tuple())) < ('2', '6'): install_requires.append('multiprocessing>=2.6') setup( name='imposm.parser', - version="1.0.3", + version="1.0.5", description='Fast and easy OpenStreetMap XML/PBF parser.', - long_description=open('README').read() + open('CHANGES').read(), + long_description=open('README.rst').read() + open('CHANGES').read(), author='Oliver Tonnhofer', author_email='o...@omniscale.de', - url='http://dev.omniscale.net/imposm.parser/', + url='http://imposm.org/docs/imposm.parser/latest/', license='Apache Software License 2.0', packages=find_packages(), namespace_packages = ['imposm'], -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/imposm-parser.git _______________________________________________ Pkg-grass-devel mailing list Pkg-grass-devel@lists.alioth.debian.org http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel