This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository pyosmium.
commit 3a9ac3ade3cea65af599e4ffad13e5183be8bdd5 Author: Bas Couwenberg <[email protected]> Date: Thu Aug 31 23:40:27 2017 +0200 New upstream version 2.13.0 --- .travis.yml | 19 +++- CHANGELOG.md | 39 ++++--- README.md | 2 +- appveyor.yml | 4 +- lib/generic_handler.hpp | 14 +-- lib/merged_input.hpp | 42 ++++++- lib/osm.cc | 22 +++- lib/osmium.cc | 10 +- src/osmium/osm/__init__.py | 56 ++++++++++ src/osmium/replication/server.py | 19 +++- src/osmium/version.py | 6 +- test/helpers.py | 38 ++++++- test/test_geom.py | 23 +++- test/test_nodelist.py | 1 + test/test_osm.py | 23 ++-- test/test_replication.py | 229 +++++++++++++++++++++++++++++++++++++++ test/test_taglist.py | 3 +- tools/pyosmium-get-changes | 3 + 18 files changed, 489 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a275d5..88006e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ language: cpp sudo: false +dist: trusty matrix: include: @@ -25,7 +26,7 @@ matrix: - os: osx osx_image: xcode6.4 compiler: clang - env: USE_PYTHON_VERSION=2 PYTHON_SUFFIX= BOOST_PYTHON_OPTION= + env: USE_PYTHON_VERSION=2 PYTHON_SUFFIX=2 BOOST_PYTHON_OPTION= - os: osx osx_image: xcode6.4 compiler: clang @@ -44,14 +45,15 @@ addons: apt: sources: - boost-latest - - ubuntu-toolchain-r-test packages: - g++-4.8 - gcc-4.8 - libboost-python1.55-dev - libboost1.55-dev - libsparsehash-dev + - python-dev - python-nose + - python-mock - python3 - python3-dev - python3-nose @@ -62,7 +64,7 @@ install: - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then brew install python$PYTHON_SUFFIX google-sparsehash; brew install boost-python $BOOST_PYTHON_OPTION; - pip$PYTHON_SUFFIX install -q nose; + pip$PYTHON_SUFFIX install -q nose mock; fi script: @@ -70,8 +72,13 @@ script: CXX=g++-4.8; CC=gcc-4.8; fi - - python${USE_PYTHON_VERSION} --version - - python${USE_PYTHON_VERSION} setup.py build + - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then + PYTHON=python${USE_PYTHON_VERSION}; + else + PYTHON=/usr/bin/python${USE_PYTHON_VERSION}; + fi + - $PYTHON --version + - $PYTHON setup.py build - cd test - - python${USE_PYTHON_VERSION} run_tests.py + - $PYTHON run_tests.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4467604..1da72ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,26 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [2.13.0] - 2017-08-31 + +### Added + +- tests for WKB factories and replication server +- str() and repr() implementations for all classes in osmium.osm +- when applying diffs to a handler, a location cache may be used + +### Changed + +- use new MultipolygonManager for building areas +- allow to access nodes in a NodeRefList with negative index +- use current libosmium + +### Fixed + +- pyosmium-get-changes exits with an error when no start sequence can + be found + + ## [2.12.4] - 2017-08-19 ### Added @@ -21,7 +41,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - make apply_reader_simple a template again -- minor fised to documentation +- minor fixes to documentation + ## [2.12.3] - 2017-05-25 @@ -201,20 +222,4 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Exception not caught in test. -[unreleased]: https://github.com/osmcode/pyosmium/compare/v2.12.3...HEAD -[2.12.3]: https://github.com/osmcode/pyosmium/compare/v2.12.2...v2.12.3 -[2.12.2]: https://github.com/osmcode/pyosmium/compare/v2.12.1...v2.12.2 -[2.12.1]: https://github.com/osmcode/pyosmium/compare/v2.12.0...v2.12.1 -[2.12.0]: https://github.com/osmcode/pyosmium/compare/v2.11.0...v2.12.0 -[2.11.0]: https://github.com/osmcode/pyosmium/compare/v2.10.2...v2.11.0 -[2.10.2]: https://github.com/osmcode/pyosmium/compare/v2.9.0...v2.10.2 -[2.9.0]: https://github.com/osmcode/pyosmium/compare/v2.8.0...v2.9.0 -[2.8.0]: https://github.com/osmcode/pyosmium/compare/v2.7.1...v2.8.0 -[2.7.1]: https://github.com/osmcode/pyosmium/compare/v2.6.0...v2.7.1 -[2.6.0]: https://github.com/osmcode/pyosmium/compare/v2.5.4...v2.6.0 -[2.5.4]: https://github.com/osmcode/pyosmium/compare/v2.5.3...v2.5.4 -[2.5.3]: https://github.com/osmcode/pyosmium/compare/v2.4.1...v2.5.3 -[2.4.1]: https://github.com/osmcode/pyosmium/compare/v2.3.0...v2.4.1 -[2.3.0]: https://github.com/osmcode/pyosmium/compare/v2.2.0...v2.3.0 -[2.2.0]: https://github.com/osmcode/pyosmium/compare/v2.1.0...v2.2.0 diff --git a/README.md b/README.md index 2a56bb6..fcfb39f 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ There is a small test suite in the test directory. This provides regression test for the python bindings, it is not meant to be a test suite for Libosmium. You'll need the Python `nose` module. On Debian/Ubuntu install the package -`python-nose`. +`python-nose`. For Python2 `mock` is required as well (package `python-mock`). The suite can be run with: diff --git a/appveyor.yml b/appveyor.yml index 8ce1bfa..b0a35c4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,8 +3,10 @@ environment: matrix: - PYTHON: "C:\\Python27-x64" DEVPACK: pyosmium_libs27-b63.7z + PIPINSTALLS: nose wheel mock - PYTHON: "C:\\Python36-x64" DEVPACK: pyosmium_libs36-b63.7z + PIPINSTALLS: nose wheel os: Visual Studio 2015 @@ -24,7 +26,7 @@ install: - 7z x c:\dev\%DEVPACK% - dir c:\libs\include - dir c:\libs\lib - - pip install nose wheel + - pip install %PIPINSTALLS% # scripts that are called at very beginning, before repo cloning init: diff --git a/lib/generic_handler.hpp b/lib/generic_handler.hpp index beaa7df..61a5ec8 100644 --- a/lib/generic_handler.hpp +++ b/lib/generic_handler.hpp @@ -2,7 +2,7 @@ #define PYOSMIUM_GENERIC_HANDLER_HPP #include <osmium/area/assembler.hpp> -#include <osmium/area/multipolygon_collector.hpp> +#include <osmium/area/multipolygon_manager.hpp> #include <osmium/handler.hpp> #include <osmium/handler/node_locations_for_ways.hpp> #include <osmium/index/map/all.hpp> @@ -44,7 +44,7 @@ void apply_with_location(osmium::io::Reader &r, const std::string &idx) { } void apply_with_area(osmium::io::Reader &r, - osmium::area::MultipolygonCollector<osmium::area::Assembler> &collector, + osmium::area::MultipolygonManager<osmium::area::Assembler> &mp_manager, const std::string &idx) { const auto& map_factory = osmium::index::MapFactory<osmium::unsigned_object_id_type, osmium::Location>::instance(); std::unique_ptr<index_type> index = map_factory.create_map(idx); @@ -52,7 +52,7 @@ void apply_with_area(osmium::io::Reader &r, location_handler.ignore_errors(); osmium::apply(r, location_handler, *this, - collector.handler([this](const osmium::memory::Buffer& area_buffer) { + mp_manager.handler([this](const osmium::memory::Buffer& area_buffer) { osmium::apply(area_buffer, *this); }) ); @@ -82,14 +82,12 @@ void apply(const osmium::io::File &file, osmium::osm_entity_bits::type types, case area_handler: { osmium::area::Assembler::config_type assembler_config; - osmium::area::MultipolygonCollector<osmium::area::Assembler> collector(assembler_config); + osmium::area::MultipolygonManager<osmium::area::Assembler> mp_manager{assembler_config}; - osmium::io::Reader reader1(file); - collector.read_relations(reader1); - reader1.close(); + osmium::relations::read_relations(file, mp_manager); osmium::io::Reader reader2(file); - apply_with_area(reader2, collector, idx); + apply_with_area(reader2, mp_manager, idx); reader2.close(); break; } diff --git a/lib/merged_input.hpp b/lib/merged_input.hpp index 4d88575..c844fd5 100644 --- a/lib/merged_input.hpp +++ b/lib/merged_input.hpp @@ -10,6 +10,8 @@ #include <osmium/io/any_output.hpp> #include <osmium/io/output_iterator.hpp> #include <osmium/handler.hpp> +#include <osmium/handler/node_locations_for_ways.hpp> +#include <osmium/index/map/all.hpp> #include <osmium/object_pointer_collection.hpp> #include <osmium/visitor.hpp> @@ -50,8 +52,7 @@ namespace { namespace pyosmium { class MergeInputReader { -public: - void apply(BaseHandler& handler, bool simplify = true) { + void apply_without_location(BaseHandler& handler, bool simplify = true) { handler.apply_start(); if (simplify) { objects.sort(osmium::object_order_type_id_reverse_version()); @@ -73,6 +74,43 @@ public: changes.clear(); } + void apply_with_location(BaseHandler& handler, const std::string &idx, + bool simplify = true) { + const auto& map_factory = osmium::index::MapFactory<osmium::unsigned_object_id_type, osmium::Location>::instance(); + std::unique_ptr<index_type> index = map_factory.create_map(idx); + osmium::handler::NodeLocationsForWays<index_type> location_handler(*index); + location_handler.ignore_errors(); + + handler.apply_start(); + if (simplify) { + objects.sort(osmium::object_order_type_id_reverse_version()); + osmium::item_type prev_type = osmium::item_type::undefined; + osmium::object_id_type prev_id = 0; + for (auto &item: objects) { + if (item.type() != prev_type || item.id() != prev_id) { + prev_type = item.type(); + prev_id = item.id(); + osmium::apply_item(item, location_handler, handler); + } + } + } else { + objects.sort(osmium::object_order_type_id_version()); + osmium::apply(objects.begin(), objects.end(), location_handler, handler); + } + + objects = osmium::ObjectPointerCollection(); + changes.clear(); + } + +public: + void apply(BaseHandler& handler, const std::string &idx = "", + bool simplify = true) { + if (idx.empty()) + apply_without_location(handler, simplify); + else + apply_with_location(handler, idx, simplify); + } + void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer, bool with_history = true) { auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(reader); diff --git a/lib/osm.cc b/lib/osm.cc index 3491237..9263381 100644 --- a/lib/osm.cc +++ b/lib/osm.cc @@ -29,6 +29,18 @@ inline const char *get_tag_by_key_with_none(osmium::TagList const& obj, return key ? obj.get_value_by_key(key) : nullptr; } +inline const osmium::NodeRef &get_item_nodereflist(osmium::NodeRefList const& list, ssize_t idx) +{ + auto sz = list.size(); + osmium::NodeRefList::size_type iout = (idx >= 0 ? idx : (ssize_t) sz + idx); + + if (iout >= sz) { + PyErr_SetString(PyExc_IndexError, "Bad index."); + boost::python::throw_error_already_set(); + } + + return list[iout]; +} inline bool taglist_contains_tag(osmium::TagList const& obj, const char *key) { @@ -88,6 +100,12 @@ BOOST_PYTHON_MODULE(_osm) .def("valid", &osmium::Location::valid, args("self"), "Check that the location is a valid WGS84 coordinate, i.e. " "that it is within the usual bounds.") + .def("lat_without_check", &osmium::Location::lat_without_check, args("self"), + "Return latitude (y coordinate) without checking if the location " + "is valid.") + .def("lon_without_check", &osmium::Location::lon_without_check, args("self"), + "Return longitude (x coordinate) without checking if the location " + "is valid.") ; class_<osmium::Box>("Box", "A bounding box around a geographic area. It is defined by an " @@ -184,8 +202,8 @@ BOOST_PYTHON_MODULE(_osm) "is normally not used directly, use one of its subclasses instead.", no_init) .def("__len__", &osmium::NodeRefList::size) - .def("__getitem__", - make_function(static_cast<const osmium::NodeRef& (osmium::NodeRefList::*)(osmium::NodeRefList::size_type) const>(&osmium::NodeRefList::operator[]), return_value_policy<reference_existing_object>())) + .def("__getitem__", make_function(&get_item_nodereflist, + return_value_policy<reference_existing_object>())) .def("__iter__", iterator<osmium::NodeRefList,return_internal_reference<>>()) .def("is_closed", &osmium::NodeRefList::is_closed, args("self"), "True if the start and end node are the same (synonym for " diff --git a/lib/osmium.cc b/lib/osmium.cc index 62d787f..0398b13 100644 --- a/lib/osmium.cc +++ b/lib/osmium.cc @@ -174,10 +174,16 @@ BOOST_PYTHON_MODULE(_osmium) "Collects data from multiple input files and sorts and optionally " "deduplicates the data before applying it to a handler.") .def("apply", &pyosmium::MergeInputReader::apply, - (arg("self"), arg("handler"), arg("simplify")=true), + (arg("self"), arg("handler"), arg("idx")="", arg("simplify")=true), "Apply collected data to a handler. The data will be sorted first. " "If `simplify` is true (default) then duplicates will be eliminated " - "and only the newest version of each object kept. After the data " + "and only the newest version of each object kept. If `idx` is given " + "a node location cache with the given type will be created and " + "applied when creating the ways. Note that a diff file normally does " + "not contain all node locations to reconstruct changed ways. If the " + "full way geometries are needed, create a persistent node location " + "cache during initial import of the area and reuse it when processing " + "diffs. After the data " "has been applied the buffer of the MergeInputReader is empty and " "new data can be added for the next round of application.") .def("apply_to_reader", &pyosmium::MergeInputReader::apply_to_reader, diff --git a/src/osmium/osm/__init__.py b/src/osmium/osm/__init__.py index 22712b4..c988227 100644 --- a/src/osmium/osm/__init__.py +++ b/src/osmium/osm/__init__.py @@ -25,3 +25,59 @@ def create_mutable_relation(rel, **args): Node.replace = create_mutable_node Way.replace = create_mutable_way Relation.replace = create_mutable_relation + +Location.__repr__ = lambda l : 'osmium.osm.Location(x=%r, y=%r)' \ + % (l.x, l.y) \ + if l.valid() else 'osmium.osm.Location()' +Location.__str__ = lambda l : '%f/%f' % (l.lon_without_check(), l.lat_without_check()) \ + if l.valid() else 'invalid' + +Box.__repr__ = lambda b : 'osmium.osm.Box(bottom_left=%r, top_right=%r)' \ + % (b.bottom_left, b.top_right) +Box.__str__ = lambda b : '(%s %s)' % (b.bottom_left, b.top_right) + +Tag.__repr__ = lambda t : 'osmium.osm.Tag(k=%s, v=%s)' % (t.k, t.v) +Tag.__str__ = lambda t : '%s=%s' % (t.k, t.v) + +TagList.__repr__ = lambda t : "osmium.osm.TagList({%s})" \ + % ",".join(["%r=%r" % (i.k, i.v) for i in t]) +TagList.__str__ = lambda t : "{%s}" % ",".join([str(i) for i in t]) + +NodeRef.__repr__ = lambda n : 'osmium.osm.NodeRef(ref=%r, location=%r)' % (n.ref, n.location) +NodeRef.__str__ = lambda n : '%s@%s' % (n.ref, n.location) if n.location.valid() \ + else str(n.ref) + +NodeRefList.__repr__ = lambda t : "%s([%s])" % (t.__class__.__name__, + ",".join([repr(i) for i in t])) +NodeRefList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t]) + +RelationMember.__repr__ = lambda r : 'osmium.osm.RelationMember(ref=%r, type=%r, role=%r)' \ + % (r.ref, r.type, r.role) +RelationMember.__str__ = lambda r : '%s%d@%s' % (r.type, r.ref, r.role) \ + if r.role else '%s%d' % (r.type, r.ref) + +RelationMemberList.__repr__ = lambda t : "osmium.osm.RelationMemberList([%s])" \ + % ",".join([repr(i) for i in t]) +RelationMemberList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t]) + +OSMObject.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags) + +def _str_ellipse(s, length=50): + s = str(s) + return s if len(s) <= length else (s[:length - 4] + '...' + s[-1]) + +Node.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, location=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.location) +Node.__str__ = lambda n : 'n%d: location=%s tags=%s' \ + % (n.id, n.location, _str_ellipse(n.tags)) + +Way.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, nodes=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.nodes) +Way.__str__ = lambda o : 'w%d: nodes=%s tags=%s' \ + % (o.id, _str_ellipse(o.nodes), _str_ellipse(o.tags)) + +Relation.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, members=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.members) +Relation.__str__ = lambda o : 'r%d: members=%s, tags=%s' \ + % (o.id, _str_ellipse(o.members), _str_ellipse(o.tags)) + +Changeset.__repr__ = lambda o : '%s(id=%r, uid=%r, created_at=%r, closed_at=%r, open=%r, num_changes=%r, bounds=%r, user=%r, tags=%s)' %(o.__class__.__name__, o.id, o.uid, o.created_at, o.closed_at, o.open, o.num_changes, o.bounds, o.user, o.tags) +Changeset.__str__ = lambda o : 'c%d: closed_at=%s, bounds=%s, tags=%s' \ + % (o.id, o.closed_at, o.bounds, _str_ellipse(o.tags)) diff --git a/src/osmium/replication/server.py b/src/osmium/replication/server.py index 345b333..f8f6373 100644 --- a/src/osmium/replication/server.py +++ b/src/osmium/replication/server.py @@ -78,13 +78,28 @@ class ReplicationServer(object): return DownloadResult(current_id - 1, rd, newest.sequence) - def apply_diffs(self, handler, start_id, max_size=1024, simplify=True): + def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True): """ Download diffs starting with sequence id `start_id`, merge them together and then apply them to handler `handler`. `max_size` restricts the number of diffs that are downloaded. The download stops as soon as either a diff cannot be downloaded or the unpacked data in memory exceeds `max_size` kB. + If `idx` is set, a location cache will be created and applied to + the way nodes. You should be aware that diff files usually do not + contain the complete set of nodes when a way is modified. That means + that you cannot just create a new location cache, apply it to a diff + and expect to get complete way geometries back. Instead you need to + do an initial data import using a persistent location cache to + obtain a full set of node locations and then reuse this location + cache here when applying diffs. + + Diffs may contain multiple versions of the same object when it was + changed multiple times during the period covered by the diff. If + `simplify` is set to False then all versions are returned. If it + is True (the default) then only the most recent version will be + sent to the handler. + The function returns the sequence id of the last diff that was downloaded or None if the download failed completely. """ @@ -93,7 +108,7 @@ class ReplicationServer(object): if diffs is None: return None - diffs.reader.apply(handler, simplify) + diffs.reader.apply(handler, idx=idx, simplify=simplify) return diffs.id diff --git a/src/osmium/version.py b/src/osmium/version.py index 9b0bd33..0d61057 100644 --- a/src/osmium/version.py +++ b/src/osmium/version.py @@ -3,9 +3,9 @@ Version information. """ # the major version -pyosmium_major = '2.12' +pyosmium_major = '2.13' # current release (Pip version) -pyosmium_release = '2.12.4' +pyosmium_release = '2.13.0' # libosmium version shipped with the Pip release -libosmium_version = '2.12.2' +libosmium_version = '2.13.1' diff --git a/test/helpers.py b/test/helpers.py index df0862d..13ea5b2 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -3,8 +3,20 @@ import random import tempfile import os +import sys from textwrap import dedent import osmium +from datetime import datetime + +if sys.version_info[0] == 3: + from datetime import timezone + + def mkdate(*args): + return datetime(*args, tzinfo=timezone.utc) +else: + def mkdate(*args): + return datetime(*args) + def _complete_object(o): """Takes a hash with an incomplete OSM object description and returns a @@ -102,6 +114,8 @@ def osmobj(kind, **args): ret['type'] = kind return ret +def check_repr(o): + return not str(o).startswith('<') and not repr(o).startswith('<') class HandlerTestBase: @@ -115,7 +129,29 @@ class HandlerTestBase: fn = create_opl_file(self.data) try: - self.Handler().apply_file(fn, self.apply_locations, self.apply_idx) + self.handler = self.Handler() + self.handler.apply_file(fn, self.apply_locations, self.apply_idx) finally: os.remove(fn) + if hasattr(self, "check_result"): + self.check_result() + + +class CountingHandler(osmium.SimpleHandler): + + def __init__(self): + super(CountingHandler, self).__init__() + self.counts = [0, 0, 0, 0] + + def node(self, _): + self.counts[0] += 1 + + def way(self, _): + self.counts[1] += 1 + + def relation(self, _): + self.counts[2] += 1 + + def area(self, _): + self.counts[3] += 1 diff --git a/test/test_geom.py b/test/test_geom.py index 74bdd84..1c4e9bc 100644 --- a/test/test_geom.py +++ b/test/test_geom.py @@ -10,8 +10,12 @@ class TestWkbCreateNode(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1)] class Handler(o.SimpleHandler): + wkbs = [] def node(self, n): - wkb = wkbfab.create_point(n) + self.wkbs.append(wkbfab.create_point(n)) + + def check_result(self): + assert_equals(1, len(self.handler.wkbs)) class TestWkbCreateWay(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, lat=0, lon=0), @@ -21,10 +25,14 @@ class TestWkbCreateWay(HandlerTestBase, unittest.TestCase): apply_locations = True class Handler(o.SimpleHandler): + wkbs = [] def way(self, w): - wkb = wkbfab.create_linestring(w) - wkb = wkbfab.create_linestring(w, direction=o.geom.direction.BACKWARD) - wkb = wkbfab.create_linestring(w, use_nodes=o.geom.use_nodes.ALL) + self.wkbs.append(wkbfab.create_linestring(w)) + self.wkbs.append(wkbfab.create_linestring(w, direction=o.geom.direction.BACKWARD)) + self.wkbs.append(wkbfab.create_linestring(w, use_nodes=o.geom.use_nodes.ALL)) + + def check_result(self): + assert_equals(3, len(self.handler.wkbs)) class TestWkbCreatePoly(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, lat=0, lon=0), @@ -36,5 +44,10 @@ class TestWkbCreatePoly(HandlerTestBase, unittest.TestCase): apply_locations = True class Handler(o.SimpleHandler): + wkbs = [] + def area(self, a): - wkb = wkbfab.create_multipolygon(a) + self.wkbs.append(wkbfab.create_multipolygon(a)) + + def check_result(self): + assert_equals(1, len(self.handler.wkbs)) diff --git a/test/test_nodelist.py b/test/test_nodelist.py index 46937a2..362abed 100644 --- a/test/test_nodelist.py +++ b/test/test_nodelist.py @@ -32,6 +32,7 @@ class TestNodeIds(HandlerTestBase, unittest.TestCase): eq_(34359737784, w.nodes[2].ref) eq_(-34, w.nodes[3].ref) eq_(0, w.nodes[4].ref) + eq_(0, w.nodes[-1].ref) class TestMissingRef(HandlerTestBase, unittest.TestCase): data = """\ diff --git a/test/test_osm.py b/test/test_osm.py index ea5c2c6..288196c 100644 --- a/test/test_osm.py +++ b/test/test_osm.py @@ -2,19 +2,7 @@ from nose.tools import * import unittest import os import sys -from datetime import datetime - -if sys.version_info[0] == 3: - from datetime import timezone - - def mkdate(*args): - return datetime(*args, tzinfo=timezone.utc) -else: - def mkdate(*args): - return datetime(*args) - - -from helpers import create_osm_file, osmobj, HandlerTestBase +from helpers import create_osm_file, osmobj, check_repr, HandlerTestBase, mkdate import osmium as o @@ -23,6 +11,7 @@ class TestLocation(unittest.TestCase): def test_invalid_location(self): loc = o.osm.Location() assert_false(loc.valid()) + assert_true(check_repr(loc)) def test_valid_location(self): loc = o.osm.Location(1,10) @@ -30,6 +19,8 @@ class TestLocation(unittest.TestCase): assert_equals(loc.lat, 10, 0.00001) assert_equals(loc.x, 10000000) assert_equals(loc.y, 100000000) + assert_true(check_repr(loc)) + class TestNodeAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, version=5, changeset=58674, uid=42, @@ -47,6 +38,7 @@ class TestNodeAttributes(HandlerTestBase, unittest.TestCase): assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35)) assert_equals(n.user, 'anonymous') assert_equals(n.positive_id(), 1) + assert_true(check_repr(n)) class TestNodePositiveId(HandlerTestBase, unittest.TestCase): @@ -91,6 +83,8 @@ class TestWayAttributes(HandlerTestBase, unittest.TestCase): assert_false(n.is_closed()) assert_false(n.ends_have_same_id()) assert_false(n.ends_have_same_location()) + assert_true(check_repr(n)) + assert_true(check_repr(n.nodes)) class TestRelationAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('R', id=1, version=5, changeset=58674, uid=42, @@ -109,6 +103,8 @@ class TestRelationAttributes(HandlerTestBase, unittest.TestCase): assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35)) assert_equals(n.user, 'anonymous') assert_equals(n.positive_id(), 1) + assert_true(check_repr(n)) + assert_true(check_repr(n.members)) class TestAreaFromWayAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, lat=0, lon=0), @@ -167,3 +163,4 @@ class TestChangesetAttributes(HandlerTestBase, unittest.TestCase): assert_equals(515288620, c.bounds.top_right.y) assert_equals(-1465242, c.bounds.bottom_left.x) assert_equals(515288506, c.bounds.bottom_left.y) + assert_true(check_repr(c)) diff --git a/test/test_replication.py b/test/test_replication.py new file mode 100644 index 0000000..9ca1fa0 --- /dev/null +++ b/test/test_replication.py @@ -0,0 +1,229 @@ +from nose.tools import * +import unittest +from io import BytesIO +from textwrap import dedent +from helpers import mkdate, CountingHandler + +try: + from urllib.error import URLError +except ImportError: + from urllib2 import URLError + +try: + from unittest.mock import MagicMock, patch +except ImportError: + from mock import MagicMock, patch + +import osmium as o +import osmium.replication.server as rserv + +class UrllibMock(MagicMock): + + def set_result(self, s): + self.return_value = BytesIO(dedent(s).encode()) + + def set_script(self, files): + self.side_effect = [BytesIO(dedent(s).encode()) for s in files] + +def test_get_state_url(): + svr = rserv.ReplicationServer("http://text.org";) + + data = [ + (None, 'http://text.org/state.txt'), + (1, 'http://text.org/000/000/001.state.txt'), + (999, 'http://text.org/000/000/999.state.txt'), + (1000, 'http://text.org/000/001/000.state.txt'), + (573923, 'http://text.org/000/573/923.state.txt'), + (3290012, 'http://text.org/003/290/012.state.txt'), + ] + + for i, o in data: + assert_equals(o, svr.get_state_url(i)) + +def test_get_diff_url(): + svr = rserv.ReplicationServer("https://who.is/me/";) + + data = [ + (1, 'https://who.is/me//000/000/001.osc.gz'), + (500, 'https://who.is/me//000/000/500.osc.gz'), + (83750, 'https://who.is/me//000/083/750.osc.gz'), + (999999999, 'https://who.is/me//999/999/999.osc.gz'), + ] + + for i, o in data: + assert_equals(o, svr.get_diff_url(i)) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_get_state_valid(mock): + mock.set_result("""\ + #Sat Aug 26 11:04:04 UTC 2017 + txnMaxQueried=1219304113 + sequenceNumber=2594669 + timestamp=2017-08-26T11\:04\:02Z + txnReadyList= + txnMax=1219304113 + txnActiveList=1219303583,1219304054,1219304104""") + + res = rserv.ReplicationServer("http://test.io";).get_state_info() + + assert_is_not_none(res) + assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2)) + assert_equals(res.sequence, 2594669) + + assert_equal(mock.call_count, 1) + +@patch('osmium.replication.server.urlrequest.urlopen') +def test_get_state_server_timeout(mock): + mock.side_effect = URLError(reason='Mock') + + svr = rserv.ReplicationServer("http://test.io";) + assert_is_none(svr.get_state_info()) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_diffs_count(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 + w1 + r1 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + h = CountingHandler() + assert_equals(100, svr.apply_diffs(h, 100, 10000)) + + assert_equals(h.counts, [1, 1, 1, 0]) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_diffs_without_simplify(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 v23 + n1 v24 + w1 + r1 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + h = CountingHandler() + assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=False)) + assert_equals([2, 1, 1, 0], h.counts) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_diffs_with_simplify(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 v23 + n1 v24 + w1 + r1 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + h = CountingHandler() + assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=True)) + assert_equals([1, 1, 1, 0], h.counts) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_with_location(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 x10.0 y23.0 + w1 Nn1,n2 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + class Handler(CountingHandler): + def way(self, w): + self.counts[1] += 1 + assert_equals(2, len(w.nodes)) + assert_equals(1, w.nodes[0].ref) + assert_equals(10, w.nodes[0].location.lon) + assert_equals(23, w.nodes[0].location.lat) + assert_equals(2, w.nodes[1].ref) + assert_false(w.nodes[1].location.valid()) + + h = Handler() + assert_equals(100, svr.apply_diffs(h, 100, 10000, idx="flex_mem")) + + assert_equals(h.counts, [1, 1, 0, 0]) + + + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_reader_without_simplify(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 v23 + n1 v24 + w1 + r1 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + h = CountingHandler() + + diffs = svr.collect_diffs(100, 100000) + assert_is_not_none(diffs) + + diffs.reader.apply(h, simplify=False) + assert_equals([2, 1, 1, 0], h.counts) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_reader_with_simplify(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 v23 + n1 v24 + w1 + r1 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + h = CountingHandler() + diffs = svr.collect_diffs(100, 100000) + assert_is_not_none(diffs) + + diffs.reader.apply(h, simplify=True) + assert_equals([1, 1, 1, 0], h.counts) + +@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) +def test_apply_reader_with_location(mock): + mock.set_script(("""\ + sequenceNumber=100 + timestamp=2017-08-26T11\:04\:02Z + """, """ + n1 x10.0 y23.0 + w1 Nn1,n2 + """)) + svr = rserv.ReplicationServer("http://test.io";, "opl") + + class Handler(CountingHandler): + def way(self, w): + self.counts[1] += 1 + assert_equals(2, len(w.nodes)) + assert_equals(1, w.nodes[0].ref) + assert_equals(10, w.nodes[0].location.lon) + assert_equals(23, w.nodes[0].location.lat) + assert_equals(2, w.nodes[1].ref) + assert_false(w.nodes[1].location.valid()) + + h = Handler() + diffs = svr.collect_diffs(100, 100000) + assert_is_not_none(diffs) + + diffs.reader.apply(h, idx="flex_mem") + + assert_equals(h.counts, [1, 1, 0, 0]) diff --git a/test/test_taglist.py b/test/test_taglist.py index 0437644..c59f4cf 100644 --- a/test/test_taglist.py +++ b/test/test_taglist.py @@ -4,7 +4,7 @@ import os import sys from datetime import datetime -from helpers import create_osm_file, osmobj, HandlerTestBase +from helpers import create_osm_file, osmobj, check_repr, HandlerTestBase import osmium as o @@ -72,6 +72,7 @@ class TestTagContains(HandlerTestBase, unittest.TestCase): assert_not_in("x", n.tags) assert_not_in(None, n.tags) assert_not_in("", n.tags) + assert_true(check_repr(n.tags)) class TestTagIndexOp(HandlerTestBase, unittest.TestCase): data = "n234 Tabba=x,2=vvv,xx=abba" diff --git a/tools/pyosmium-get-changes b/tools/pyosmium-get-changes index dac4403..f86ffe3 100755 --- a/tools/pyosmium-get-changes +++ b/tools/pyosmium-get-changes @@ -188,6 +188,9 @@ if __name__ == '__main__': svr = rserv.ReplicationServer(url) startseq = options.start.get_sequence(svr) + if startseq is None: + log.error("Cannot read state file from server. Is the URL correct?") + exit(1) if options.outfile is None: write_end_sequence(options.seq_file, startseq) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/pyosmium.git _______________________________________________ Pkg-grass-devel mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel
