Author: jbronn
Date: 2007-11-17 15:57:12 -0600 (Sat, 17 Nov 2007)
New Revision: 6687
Added:
django/branches/gis/django/contrib/gis/tests/layermap/
django/branches/gis/django/contrib/gis/tests/layermap/__init__.py
django/branches/gis/django/contrib/gis/tests/layermap/cities/
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.dbf
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.prj
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.shp
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.shx
django/branches/gis/django/contrib/gis/tests/layermap/interstates/
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.dbf
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.prj
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.shp
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.shx
django/branches/gis/django/contrib/gis/tests/layermap/models.py
django/branches/gis/django/contrib/gis/tests/layermap/tests.py
Modified:
django/branches/gis/django/contrib/gis/tests/__init__.py
django/branches/gis/django/contrib/gis/tests/geoapp/tests.py
django/branches/gis/django/contrib/gis/utils/layermapping.py
django/branches/gis/django/contrib/gis/utils/ogrinfo.py
Log:
gis: LayerMapping refactor
(1) Moved all routines into LayerMapping class (for easier subclassing) and
modularized the routines.
(2) OFTString and OFTReal OGR fields are verified w/the Django fields prior to
insertion, thus avoiding invalidating a large transaction.
(3) Added keyword options for specifying the transaction mode, not performing
transformations, and status printing.
(4) Created unit tests.
Other Changes:
Updated `ogrinfo` for GDAL refactor and fixed an iterating bug; simplified a
few lines in `geoapp` model tests.
Modified: django/branches/gis/django/contrib/gis/tests/__init__.py
===================================================================
--- django/branches/gis/django/contrib/gis/tests/__init__.py 2007-11-17
21:38:36 UTC (rev 6686)
+++ django/branches/gis/django/contrib/gis/tests/__init__.py 2007-11-17
21:57:12 UTC (rev 6687)
@@ -2,14 +2,21 @@
from copy import copy
from unittest import TestSuite, TextTestRunner
from django.contrib.gis.gdal import HAS_GDAL
-from django.contrib.gis.tests.utils import mysql
+try:
+ from django.contrib.gis.tests.utils import mysql
+except:
+ mysql = False
+# Tests that require use of a spatial database (e.g., creation of models)
+test_models = ['geoapp']
+
# Tests that do not require setting up and tearing down a spatial database.
test_suite_names = [
'test_geos',
'test_measure',
]
if HAS_GDAL:
+ test_models += ['layermap']
test_suite_names += [
'test_gdal_driver',
'test_gdal_ds',
@@ -21,8 +28,6 @@
else:
print >>sys.stderr, "GDAL not available - no GDAL tests will be run."
-test_models = ['geoapp']
-
def suite():
"Builds a test suite for the GIS package."
s = TestSuite()
@@ -40,8 +45,8 @@
Run the tests that require creation of a spatial database.
In order to run geographic model tests the DATABASE_USER will require
- superuser priviliges. To accomplish this outside the `postgres` user,
- create your own PostgreSQL database as a user:
+ superuser priviliges. To accomplish this outside the `postgres` user,
+ create your own PostgreSQL database as a user:
(1) Initialize database: `initdb -D /path/to/user/db`
(2) If there's already a Postgres instance on the machine, it will need
to use a different TCP port than 5432. Edit postgresql.conf (in
@@ -49,21 +54,21 @@
(3) Start this database `pg_ctl -D /path/to/user/db start`
On Windows platforms simply use the pgAdmin III utility to add superuser
- priviliges to your database user.
+ priviliges to your database user.
Make sure your settings.py matches the settings of the user database.
- For example, set the same port number (`DATABASE_PORT=5433`).
- DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER.
+ For example, set the same port number (`DATABASE_PORT=5433`).
+ DATABASE_NAME or TEST_DATABSE_NAME must be set, along with DATABASE_USER.
In settings.py set TEST_RUNNER='django.contrib.gis.tests.run_tests'.
Finally, this assumes that the PostGIS SQL files (lwpostgis.sql and
- spatial_ref_sys.sql) are installed in the directory specified by
- `pg_config --sharedir` (and defaults to /usr/local/share if that fails).
- This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings.
+ spatial_ref_sys.sql) are installed in the directory specified by
+ `pg_config --sharedir` (and defaults to /usr/local/share if that fails).
+ This behavior is overridden if `POSTGIS_SQL_PATH` is in your settings.
- Windows users should use the POSTGIS_SQL_PATH because the output
- of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'.
+ Windows users should set POSTGIS_SQL_PATH manually because the output
+ of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'.
Finally, the tests may be run by invoking `./manage.py test`.
"""
@@ -93,9 +98,10 @@
tsuite = getattr(__import__('django.contrib.gis.tests.%s' %
test_model, globals(), locals(), [test_module_name]), test_module_name)
test_suite.addTest(tsuite.suite())
- # Resetting the loaded flag to take into account what we appended to the
INSTALLED_APPS
- # (since this routine is invoked through django/core/management, it
caches the apps,
- # this ensures that syncdb will see our appended models)
+ # Resetting the loaded flag to take into account what we appended to
+ # the INSTALLED_APPS (since this routine is invoked through
+ # django/core/management, it caches the apps; this ensures that syncdb
+ # will see our appended models)
from django.db.models import loading
loading._loaded = False
Modified: django/branches/gis/django/contrib/gis/tests/geoapp/tests.py
===================================================================
--- django/branches/gis/django/contrib/gis/tests/geoapp/tests.py
2007-11-17 21:38:36 UTC (rev 6686)
+++ django/branches/gis/django/contrib/gis/tests/geoapp/tests.py
2007-11-17 21:57:12 UTC (rev 6687)
@@ -77,7 +77,7 @@
inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40))
# Creating a State object using a built Polygon
- ply = Polygon(shell.clone(), inner.clone())
+ ply = Polygon(shell, inner)
nullstate = State(name='NullState', poly=ply)
self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None
nullstate.save()
@@ -94,12 +94,12 @@
# Changing the interior ring on the poly attribute.
new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30), (30,
30))
- nullstate.poly[1] = new_inner.clone()
+ ns.poly[1] = new_inner
ply[1] = new_inner
- self.assertEqual(4326, nullstate.poly.srid)
- nullstate.save()
+ self.assertEqual(4326, ns.poly.srid)
+ ns.save()
self.assertEqual(ply, State.objects.get(name='NullState').poly)
- nullstate.delete()
+ ns.delete()
@no_oracle # Oracle does not support KML.
def test03a_kml(self):
Added: django/branches/gis/django/contrib/gis/tests/layermap/__init__.py
===================================================================
Added: django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.dbf
===================================================================
(Binary files differ)
Property changes on:
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.dbf
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.prj
===================================================================
--- django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.prj
(rev 0)
+++ django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.prj
2007-11-17 21:57:12 UTC (rev 6687)
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
Added: django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.shp
===================================================================
(Binary files differ)
Property changes on:
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.shp
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.shx
===================================================================
(Binary files differ)
Property changes on:
django/branches/gis/django/contrib/gis/tests/layermap/cities/cities.shx
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.dbf
===================================================================
(Binary files differ)
Property changes on:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.dbf
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.prj
===================================================================
---
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.prj
(rev 0)
+++
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.prj
2007-11-17 21:57:12 UTC (rev 6687)
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
Added:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.shp
===================================================================
(Binary files differ)
Property changes on:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.shp
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.shx
===================================================================
(Binary files differ)
Property changes on:
django/branches/gis/django/contrib/gis/tests/layermap/interstates/interstates.shx
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: django/branches/gis/django/contrib/gis/tests/layermap/models.py
===================================================================
--- django/branches/gis/django/contrib/gis/tests/layermap/models.py
(rev 0)
+++ django/branches/gis/django/contrib/gis/tests/layermap/models.py
2007-11-17 21:57:12 UTC (rev 6687)
@@ -0,0 +1,29 @@
+from django.contrib.gis.db import models
+
+class City(models.Model):
+ name = models.CharField(max_length=25)
+ population = models.IntegerField()
+ density = models.DecimalField(max_digits=7, decimal_places=1)
+ date = models.DateField()
+ point = models.PointField()
+ objects = models.GeoManager()
+
+class Interstate(models.Model):
+ name = models.CharField(max_length=20)
+ length = models.DecimalField(max_digits=7, decimal_places=2)
+ path = models.LineStringField()
+ objects = models.GeoManager()
+
+# Mapping dictionary for the City model.
+city_mapping = {'name' : 'Name',
+ 'population' : 'Population',
+ 'density' : 'Density',
+ 'date' : 'Created',
+ 'point' : 'POINT',
+ }
+
+# Mapping dictionary for the Interstate model.
+inter_mapping = {'name' : 'Name',
+ 'length' : 'Length',
+ 'path' : 'LINESTRING',
+ }
Added: django/branches/gis/django/contrib/gis/tests/layermap/tests.py
===================================================================
--- django/branches/gis/django/contrib/gis/tests/layermap/tests.py
(rev 0)
+++ django/branches/gis/django/contrib/gis/tests/layermap/tests.py
2007-11-17 21:57:12 UTC (rev 6687)
@@ -0,0 +1,106 @@
+import os, unittest
+from copy import copy
+from datetime import date
+from decimal import Decimal
+from models import City, Interstate, city_mapping, inter_mapping
+from django.contrib.gis.utils.layermapping import LayerMapping, LayerMapError,
InvalidDecimal
+from django.contrib.gis.gdal import DataSource
+
+shp_path = os.path.dirname(__file__)
+city_shp = os.path.join(shp_path, 'cities/cities.shp')
+inter_shp = os.path.join(shp_path, 'interstates/interstates.shp')
+
+class LayerMapTest(unittest.TestCase):
+
+ def test01_init(self):
+ "Testing LayerMapping initialization."
+
+ # Model field that does not exist.
+ bad1 = copy(city_mapping)
+ bad1['foobar'] = 'FooField'
+
+ # Shapefile field that does not exist.
+ bad2 = copy(city_mapping)
+ bad2['name'] = 'Nombre'
+
+ # Nonexistent geographic field type.
+ bad3 = copy(city_mapping)
+ bad3['point'] = 'CURVE'
+
+ # Incrementing through the bad mapping dictionaries and
+ # ensuring that a LayerMapError is raised.
+ for bad_map in (bad1, bad2, bad3):
+ try:
+ lm = LayerMapping(City, city_shp, bad_map)
+ except LayerMapError:
+ pass
+ else:
+ self.fail('Expected a LayerMapError.')
+
+ # A LookupError should be thrown for bogus encodings.
+ try:
+ lm = LayerMapping(City, city_shp, city_mapping, encoding='foobar')
+ except LookupError:
+ pass
+ else:
+ self.fail('Expected a LookupError')
+
+ def test02_simple_layermap(self):
+ "Test LayerMapping import of a simple point shapefile."
+
+ # Setting up for the LayerMapping.
+ lm = LayerMapping(City, city_shp, city_mapping)
+ lm.save()
+
+ # There should be three cities in the shape file.
+ self.assertEqual(3, City.objects.count())
+
+ # Opening up the shapefile, and verifying the values in each
+ # of the features made it to the model.
+ ds = DataSource(city_shp)
+ layer = ds[0]
+ for feat in layer:
+ city = City.objects.get(name=feat['Name'].value)
+ self.assertEqual(feat['Population'].value, city.population)
+ self.assertEqual(Decimal(str(feat['Density'])), city.density)
+ self.assertEqual(feat['Created'].value, city.date)
+
+ # Comparing the geometries.
+ pnt1, pnt2 = feat.geom, city.point
+ self.assertAlmostEqual(pnt1.x, pnt2.x, 6)
+ self.assertAlmostEqual(pnt1.y, pnt2.y, 6)
+
+ def test03_layermap_strict(self):
+ "Testing the `strict` keyword, and import of a LineString shapefile."
+
+ # When the `strict` keyword is set an error encountered will force
+ # the importation to stop.
+ try:
+ lm = LayerMapping(Interstate, inter_shp, inter_mapping,
+ strict=True, silent=True)
+ lm.save()
+ except InvalidDecimal:
+ pass
+ else:
+ self.fail('Should have failed on strict import with invalid
decimal values.')
+
+ # This LayerMapping should work b/c `strict` is not set.
+ lm = LayerMapping(Interstate, inter_shp, inter_mapping, silent=True)
+ lm.save()
+
+ # Only one interstate should have imported correctly.
+ self.assertEqual(1, Interstate.objects.count())
+
+ # Verifying the values in the single feature w/the model.
+ ds = DataSource(inter_shp)
+ feat = ds[0][0]
+ istate = Interstate.objects.get(name=feat['Name'].value)
+ self.assertEqual(Decimal(str(feat['Length'])), istate.length)
+ for p1, p2 in zip(feat.geom, istate.path):
+ self.assertAlmostEqual(p1[0], p2[0], 6)
+ self.assertAlmostEqual(p1[1], p2[1], 6)
+
+def suite():
+ s = unittest.TestSuite()
+ s.addTest(unittest.makeSuite(LayerMapTest))
+ return s
Modified: django/branches/gis/django/contrib/gis/utils/layermapping.py
===================================================================
--- django/branches/gis/django/contrib/gis/utils/layermapping.py
2007-11-17 21:38:36 UTC (rev 6686)
+++ django/branches/gis/django/contrib/gis/utils/layermapping.py
2007-11-17 21:57:12 UTC (rev 6687)
@@ -1,21 +1,19 @@
# LayerMapping -- A Django Model/OGR Layer Mapping Utility
"""
-The LayerMapping class provides a way to map the contents of OGR
+ The LayerMapping class provides a way to map the contents of OGR
vector files (e.g. SHP files) to Geographic-enabled Django models.
-This grew out of my personal needs, specifically the code repetition
+ This grew out of my personal needs, specifically the code repetition
that went into pulling geometries and fields out of an OGR layer,
converting to another coordinate system (e.g. WGS84), and then inserting
- into a Geographic Django model.
+ into a GeoDjango model.
-This utility is still in early stages of development, so its usage
+ This utility is still in early stages of development, so its usage
is subject to change -- please report any bugs.
-TODO: Unit tests and documentation.
+ Requirements: OGR C Library (from GDAL) required.
-Requirements: OGR C Library (from GDAL) required.
-
-Usage:
+ Usage:
lm = LayerMapping(model, source_file, mapping) where,
model:
@@ -32,7 +30,7 @@
is a geographic then it should correspond to the OGR
geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'.
-Keyword Args:
+ Keyword Args:
layer:
The index of the layer to use from the Data Source (defaults to 0)
@@ -46,6 +44,26 @@
For example, 'latin-1', 'utf-8', and 'cp437' are all valid
encoding parameters.
+ check:
+ By default, LayerMapping increments through each feature in the
+ layer to ensure that it is compatible with the given model and
+ mapping. Setting this keyword to False, disables this action,
+ which will speed up execution time for very large files.
+
+ silent:
+ By default, non-fatal error notifications are printed to stdout; this
+ keyword may be set in order to disable these notifications.
+
+ strict:
+ Setting this keyword to True will instruct the save() method to
+ cease execution on the first error encountered.
+
+ transaction_mode:
+ May be 'commit_on_success' (default) or 'autocommit'.
+
+ transform:
+ Setting this to False will disable all coordinate transformations.
+
Example:
1. You need a GDAL-supported data source, like a shapefile.
@@ -91,162 +109,86 @@
Saved: Name: 3
LayerMapping just transformed the three geometries from the SHP file from
their
- source spatial reference system (WGS84) to the spatial reference system of
- the GeoDjango model (NAD83). If no spatial reference system is defined for
- the layer, use the `source_srs` keyword with a SpatialReference object to
- specify one. Further, data is selectively imported from the given data
source
- fields into the model fields.
+ source spatial reference system (WGS84) to the spatial reference system of
+ the GeoDjango model (NAD83). If no spatial reference system is defined for
+ the layer, use the `source_srs` keyword with a SpatialReference object to
+ specify one.
"""
-from types import StringType, TupleType
-from datetime import datetime
+from datetime import date, datetime
+from decimal import Decimal
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import connection, transaction
+from django.db.models.fields.related import ForeignKey
from django.contrib.gis.db.backend import SPATIAL_BACKEND
-from django.contrib.gis.gdal import \
- OGRGeometry, OGRGeomType, SpatialReference, CoordTransform, \
- DataSource, OGRException
-from django.contrib.gis.gdal.field import Field, OFTInteger, OFTReal,
OFTString, OFTDateTime
+from django.contrib.gis.gdal import CoordTransform, DataSource, \
+ OGRException, OGRGeometry, OGRGeomType, SpatialReference
+from django.contrib.gis.gdal.field import OFTDate, OFTDateTime, OFTInteger,
OFTReal, OFTString, OFTTime
from django.contrib.gis.models import GeometryColumns, SpatialRefSys
-from django.db import connection, transaction
-from django.core.exceptions import ObjectDoesNotExist
-# A mapping of given geometry types to their OGR integer type.
-ogc_types = {'POINT' : OGRGeomType('Point'),
- 'LINESTRING' : OGRGeomType('LineString'),
- 'POLYGON' : OGRGeomType('Polygon'),
- 'MULTIPOINT' : OGRGeomType('MultiPoint'),
- 'MULTILINESTRING' : OGRGeomType('MultiLineString'),
- 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'),
- 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'),
- }
+# LayerMapping exceptions.
+class LayerMapError(Exception): pass
+class InvalidString(LayerMapError): pass
+class InvalidDecimal(LayerMapError): pass
-# The django.contrib.gis model types.
-gis_fields = {'PointField' : 'POINT',
- 'LineStringField': 'LINESTRING',
- 'PolygonField': 'POLYGON',
- 'MultiPointField' : 'MULTIPOINT',
- 'MultiLineStringField' : 'MULTILINESTRING',
- 'MultiPolygonField' : 'MULTIPOLYGON',
- }
+class LayerMapping(object):
+ "A class that maps OGR Layers to GeoDjango Models."
+
+ # A mapping of given geometry types to their OGR integer type.
+ OGC_TYPES = {'POINT' : OGRGeomType('Point'),
+ 'LINESTRING' : OGRGeomType('LineString'),
+ 'POLYGON' : OGRGeomType('Polygon'),
+ 'MULTIPOINT' : OGRGeomType('MultiPoint'),
+ 'MULTILINESTRING' : OGRGeomType('MultiLineString'),
+ 'MULTIPOLYGON' : OGRGeomType('MultiPolygon'),
+ 'GEOMETRYCOLLECTION' : OGRGeomType('GeometryCollection'),
+ }
-# Acceptable 'base' types for a multi-geometry type.
-multi_types = {'POINT' : OGRGeomType('MultiPoint'),
- 'LINESTRING' : OGRGeomType('MultiLineString'),
- 'POLYGON' : OGRGeomType('MultiPolygon'),
- }
+ # The django.contrib.gis model types.
+ GIS_FIELDS = {'PointField' : 'POINT',
+ 'LineStringField': 'LINESTRING',
+ 'PolygonField': 'POLYGON',
+ 'MultiPointField' : 'MULTIPOINT',
+ 'MultiLineStringField' : 'MULTILINESTRING',
+ 'MultiPolygonField' : 'MULTIPOLYGON',
+ 'GeometryCollectionField' : 'GEOMETRYCOLLECTION',
+ }
-def map_foreign_key(django_field):
- from django.db.models.fields.related import ForeignKey
+ # Acceptable 'base' types for a multi-geometry type.
+ MULTI_TYPES = {'POINT' : OGRGeomType('MultiPoint'),
+ 'LINESTRING' : OGRGeomType('MultiLineString'),
+ 'POLYGON' : OGRGeomType('MultiPolygon'),
+ }
- if not django_field.__class__ is ForeignKey:
- return django_field.__class__.__name__
+ # The acceptable Django field types that map to OGR fields.
+ FIELD_TYPES = {
+ 'AutoField' : OFTInteger,
+ 'IntegerField' : OFTInteger,
+ 'FloatField' : OFTReal,
+ 'DateField' : OFTDate,
+ 'DateTimeField' : OFTDateTime,
+ 'TimeField' : OFTTime,
+ 'DecimalField' : OFTReal,
+ 'CharField' : OFTString,
+ 'TextField' : OFTString,
+ 'SmallIntegerField' : OFTInteger,
+ 'PositiveSmallIntegerField' : OFTInteger,
+ }
-
- rf=django_field.rel.get_related_field()
+ # The acceptable transaction modes.
+ TRANSACTION_MODES = {'autocommit' : transaction.autocommit,
+ 'commit_on_success' : transaction.commit_on_success,
+ }
- return rf.get_internal_type()
-
-# The acceptable Django field types that map to OGR fields.
-field_types = {
- 'AutoField' : OFTInteger,
- 'IntegerField' : OFTInteger,
- 'FloatField' : OFTReal,
- 'DateTimeField' : OFTDateTime,
- 'DecimalField' : OFTReal,
- 'CharField' : OFTString,
- 'SmallIntegerField' : OFTInteger,
- 'PositiveSmallIntegerField' : OFTInteger,
- }
-
-def make_multi(geom_name, model_type):
- "Determines whether the geometry should be made into a GeometryCollection."
- if (geom_name in multi_types) and (model_type.startswith('Multi')):
- return True
- else:
- return False
-
-def check_feature(feat, model_fields, mapping):
- "Checks the OGR layer feature."
-
- HAS_GEO = False
-
- # Incrementing through each model_field & ogr_field in the given mapping.
- for model_field, ogr_field in mapping.items():
-
- # Making sure the given mapping model field is in the given model
fields.
- if model_field in model_fields:
- model_type = model_fields[model_field]
- elif model_field[:-3] in model_fields: #foreign key
- model_type = model_fields[model_field[:-3]]
- else:
- raise Exception('Given mapping field "%s" not in given Model
fields!' % model_field)
-
- ### Handling if we get a geometry in the Field ###
- if ogr_field in ogc_types:
- # At this time, no more than one geographic field per model =(
- if HAS_GEO:
- raise Exception('More than one geographic field in mapping not
allowed (yet).')
- else:
- HAS_GEO = ogr_field
-
- # Making sure this geometry field type is a valid Django GIS field.
- if not model_type in gis_fields:
- raise Exception('Unknown Django GIS field type "%s"' %
model_type)
-
- # Getting the OGRGeometry, it's type (an integer) and it's name (a
string)
- geom = feat.geom
- gtype = geom.geom_type
- gname = geom.geom_name
-
- if make_multi(gname, model_type):
- # Do we have to 'upsample' into a Geometry Collection?
- pass
- elif gtype == ogc_types[gis_fields[model_type]]:
- # The geometry type otherwise was expected
- pass
- else:
- raise Exception('Invalid mapping geometry; model has %s,
feature has %s' % (model_type, gtype))
-
- ## Handling other fields
- else:
- # Making sure the model field is
- if not model_type in field_types:
- raise Exception('Django field type "%s" has no OGR mapping
(yet).' % model_type)
-
- # Otherwise, we've got an OGR Field. Making sure that an
- # index exists for the mapping OGR field.
- try:
- fi = feat.index(ogr_field)
- except:
- raise Exception('Given mapping OGR field "%s" not in given OGR
layer feature!' % ogr_field)
-
-def check_layer(layer, fields, mapping):
- "Checks the OGR layer by incrementing through and checking each feature."
- # Incrementing through each feature in the layer.
- for feat in layer:
- check_feature(feat, fields, mapping)
-
-def check_srs(layer, source_srs):
- "Checks the compatibility of the given spatial reference object."
- if isinstance(source_srs, SpatialReference):
- sr = source_srs
- elif isinstance(source_srs, SpatialRefSys):
- sr = source_srs.srs
- elif isinstance(source_srs, (int, str)):
- sr = SpatialReference(source_srs)
- else:
- sr = layer.srs
- if not sr:
- raise Exception('No source reference system defined.')
- else:
- return sr
-
-class LayerMapping:
- "A class that maps OGR Layers to Django Models."
-
- def __init__(self, model, data, mapping, layer=0, source_srs=None,
encoding=None):
+ def __init__(self, model, data, mapping, layer=0,
+ source_srs=None, encoding=None, check=True,
+ progress=False, interval=1000, strict=False, silent=False,
+ transaction_mode='commit_on_success', transform=True):
"Takes the Django model, the data source, and the mapping (dictionary)"
# Getting the field names and types from the model
- fields = dict((f.name, map_foreign_key(f)) for f in model._meta.fields)
+ self.fields = dict((f.name, self.map_foreign_key(f)) for f in
model._meta.fields)
+ self.field_classes = dict((f.name, f) for f in model._meta.fields)
+
# Getting the DataSource and its Layer
if isinstance(data, basestring):
self.ds = DataSource(data)
@@ -254,120 +196,306 @@
self.ds = data
self.layer = self.ds[layer]
- # Checking the layer -- intitialization of the object will fail if
- # things don't check out before hand.
- check_layer(self.layer, fields, mapping)
-
- # Since the layer checked out, setting the fields and the mapping.
- self.fields = fields
+ # Setting the mapping
self.mapping = mapping
+
+ # Setting the model, and getting the geometry column associated
+ # with the model (an exception will be raised if there is no
+ # geometry column).
self.model = model
- self.source_srs = check_srs(self.layer, source_srs)
+ self.geo_col = self.geometry_column()
+ # Checking the source spatial reference system, and getting
+ # the coordinate transformation object (unless the `transform`
+ # keyword is set to False)
+ self.source_srs = self.check_srs(source_srs)
+ self.transform = transform and self.coord_transform()
+
+ # Checking the layer -- intitialization of the object will fail if
+ # things don't check out before hand. This may be time-consuming,
+ # and disabled by setting the `check` keyword to False.
+ if check: self.check_layer()
+
+ # The silent, strict, progress, and interval flags.
+ self.silent = silent
+ self.strict = strict
+ self.progress = progress
+ self.interval = interval
+
# Setting the encoding for OFTString fields, if specified.
if encoding:
# Making sure the encoding exists, if not a LookupError
- # exception will be thrown.
+ # exception will be thrown.
from codecs import lookup
lookup(encoding)
self.encoding = encoding
else:
self.encoding = None
- # Either the import will work, or it won't be committed.
- @transaction.commit_on_success
- def save(self, verbose=False):
- "Runs the layer mapping on the given SHP file, and saves to the
database."
+ # Setting the transaction decorator with the function in the
+ # transaction modes dictionary.
+ if transaction_mode in self.TRANSACTION_MODES:
+ self.transaction_decorator =
self.TRANSACTION_MODES[transaction_mode]
+ self.transaction_mode = transaction_mode
+ else:
+ raise LayerMapError('Unrecognized transaction mode: %s' %
transaction_mode)
- # Getting the GeometryColumn object.
+ def check_feature(self, feat):
+ "Checks the OGR feature against the model fields and mapping."
+ HAS_GEO = False
+
+ # Incrementing through each model_field & ogr_field in the given
mapping.
+ for model_field, ogr_field in self.mapping.items():
+ # Making sure the given mapping model field is in the given model
fields.
+ if model_field in self.fields:
+ model_type = self.fields[model_field]
+ elif model_field[:-3] in self.fields: #foreign key
+ model_type = self.fields[model_field[:-3]]
+ else:
+ raise LayerMapError('Given mapping field "%s" not in given
Model fields!' % model_field)
+
+ ### Handling if we get a geometry in the Field ###
+ if ogr_field in self.OGC_TYPES:
+ # At this time, no more than one geographic field per model =(
+ if HAS_GEO:
+ raise LayerMapError('More than one geographic field in
mapping not allowed (yet).')
+ else:
+ HAS_GEO = ogr_field
+
+ # Making sure this geometry field type is a valid Django GIS
field.
+ if not model_type in self.GIS_FIELDS:
+ raise LayerMapError('Unknown Django GIS field type "%s"' %
model_type)
+
+ # Getting the OGRGeometry, it's type (an integer) and it's
name (a string)
+ geom = feat.geom
+ gtype = geom.geom_type
+ gname = geom.geom_name
+
+ if self.make_multi(gname, model_type):
+ # Do we have to 'upsample' into a Geometry Collection?
+ pass
+ elif gtype == self.OGC_TYPES[self.GIS_FIELDS[model_type]]:
+ # The geometry type otherwise was expected
+ pass
+ else:
+ raise LayerMapError('Invalid mapping geometry; model has
%s, feature has %s' % (model_type, gtype))
+ ### Handling other fields ###
+ else:
+ # Making sure the model field is supported.
+ if not model_type in self.FIELD_TYPES:
+ raise LayerMapError('Django field type "%s" has no OGR
mapping (yet).' % model_type)
+
+ # Otherwise, we've got an OGR Field. Making sure that an
+ # index exists for the mapping OGR field.
+ try:
+ fi = feat.index(ogr_field)
+ except:
+ raise LayerMapError('Given mapping OGR field "%s" not in
given OGR layer feature!' % ogr_field)
+
+ def check_layer(self):
+ "Checks every feature in this object's layer."
+ for feat in self.layer:
+ self.check_feature(feat)
+
+ def check_srs(self, source_srs):
+ "Checks the compatibility of the given spatial reference object."
+ if isinstance(source_srs, SpatialReference):
+ sr = source_srs
+ elif isinstance(source_srs, SpatialRefSys):
+ sr = source_srs.srs
+ elif isinstance(source_srs, (int, str)):
+ sr = SpatialReference(source_srs)
+ else:
+ # Otherwise just pulling the SpatialReference from the layer
+ sr = self.layer.srs
+
+ if not sr:
+ raise LayerMapError('No source reference system defined.')
+ else:
+ return sr
+
+ def coord_transform(self):
+ "Returns the coordinate transformation object."
try:
- db_table = self.model._meta.db_table
- if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper()
- gc_kwargs = {GeometryColumns.table_name_col() : db_table}
- geo_col = GeometryColumns.objects.get(**gc_kwargs)
- except:
- raise Exception('Geometry column does not exist. (did you run
syncdb?)')
+ # Getting the target spatial reference system
+ target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs
- # Getting the coordinate system needed for transformation (with
CoordTransform)
- try:
- # Getting the target spatial reference system
- target_srs = SpatialRefSys.objects.get(srid=geo_col.srid).srs
-
# Creating the CoordTransform object
- ct = CoordTransform(self.source_srs, target_srs)
+ return CoordTransform(self.source_srs, target_srs)
except Exception, msg:
- raise Exception('Could not translate between the data source and
model geometry: %s' % msg)
+ raise LayerMapError('Could not translate between the data source
and model geometry: %s' % msg)
- for feat in self.layer:
- # The keyword arguments for model construction
- kwargs = {}
+ def feature_kwargs(self, feat):
+ "Returns the keyword arguments needed for saving a feature."
+
+ # The keyword arguments for model construction.
+ kwargs = {}
- # Incrementing through each model field and the OGR field in the
mapping
- all_prepped = True
+ # The all_prepped flagged, will be set to False if there's a
+ # problem w/a ForeignKey that doesn't exist.
+ all_prepped = True
- for model_field, ogr_field in self.mapping.items():
- is_fk = False
+ # Incrementing through each model field and OGR field in the
+ # dictionary mapping.
+ for model_field, ogr_field in self.mapping.items():
+ is_fk = False
+ try:
+ model_type = self.fields[model_field]
+ except KeyError: #foreign key
+ # The -3 index is b/c foreign keys are appended w/'_id'.
+ model_type = self.fields[model_field[:-3]]
+ is_fk = True
+
+ if ogr_field in self.OGC_TYPES:
+ # Verify OGR geometry.
+ val = self.verify_geom(feat.geom, model_type)
+ else:
+ # Otherwise, verify OGR Field type.
+ val = self.verify_field(feat[ogr_field], model_field)
+
+ if is_fk:
+ # Handling if foreign key.
+ rel_obj = None
+ field_name = model_field[:-3]
try:
- model_type = self.fields[model_field]
- except KeyError: #foreign key
- model_type = self.fields[model_field[:-3]]
- is_fk = True
+ # FIXME: refactor to efficiently fetch FKs.
+ # Requires significant re-work. :-/
+ rel = self.model._meta.get_field(field_name).rel
+ rel_obj = rel.to._default_manager.get(**{('%s__exact' %
rel.field_name):val})
+ except ObjectDoesNotExist:
+ all_prepped = False
- if ogr_field in ogc_types:
- ## Getting the OGR geometry from the field
- geom = feat.geom
+ kwargs[model_field[:-3]] = rel_obj
+ else:
+ kwargs[model_field] = val
+
+ return kwargs, all_prepped
- if make_multi(geom.geom_name, model_type):
- # Constructing a multi-geometry type to contain the
single geometry
- multi_type = multi_types[geom.geom_name]
- g = OGRGeometry(multi_type)
- g.add(geom)
- else:
- g = geom
+ def verify_field(self, fld, model_field):
+ """
+ Verifies if the OGR Field contents are acceptable to the Django
+ model field. If they are, the verified value is returned,
+ otherwise the proper exception is raised.
+ """
+ field_class = self.field_classes[model_field]
+ if isinstance(fld, OFTString):
+ if self.encoding:
+ # The encoding for OGR data sources may be specified here
+ # (e.g., 'cp437' for Census Bureau boundary files).
+ val = unicode(fld.value, self.encoding)
+ else:
+ val = fld.value
+ if len(val) > field_class.max_length:
+ raise InvalidString('%s model field maximum string length
is %s, given %s characters.' %
+ (model_field, field_class.max_length,
len(val)))
+ elif isinstance(fld, OFTReal):
+ try:
+ # Creating an instance of the Decimal value to use.
+ d = Decimal(str(fld.value))
+ except:
+ raise InvalidDecimal('Could not construct decimal from: %s' %
fld)
+ dtup = d.as_tuple()
+ if len(dtup[1]) > field_class.max_digits:
+ raise InvalidDecimal('More than the maximum # of digits
encountered.')
+ elif len(dtup[1][dtup[2]:]) > field_class.decimal_places:
+ raise InvalidDecimal('More than the maximum # of decimal
places encountered.')
+ val = d
+ else:
+ val = fld.value
+ return val
- # Transforming the geometry with our Coordinate
Transformation object.
- g.transform(ct)
+ def verify_geom(self, geom, model_type):
+ "Verifies the geometry."
+ if self.make_multi(geom.geom_name, model_type):
+ # Constructing a multi-geometry type to contain the single geometry
+ multi_type = self.MULTI_TYPES[geom.geom_name]
+ g = OGRGeometry(multi_type)
+ g.add(geom)
+ else:
+ g = geom
- # Updating the keyword args with the WKT of the
transformed model.
- val = g.wkt
- else:
- ## Otherwise, this is an OGR field type
- fld = feat[ogr_field]
+ # Transforming the geometry with our Coordinate Transformation object,
+ # but only if the class variable `transform` is set w/a CoordTransform
+ # object.
+ if self.transform: g.transform(self.transform)
+
+ # Returning the WKT of the geometry.
+ return g.wkt
+
+ def geometry_column(self):
+ "Returns the GeometryColumn model associated with the geographic
column."
+ # Getting the GeometryColumn object.
+ try:
+ db_table = self.model._meta.db_table
+ if SPATIAL_BACKEND == 'oracle': db_table = db_table.upper()
+ gc_kwargs = {GeometryColumns.table_name_col() : db_table}
+ return GeometryColumns.objects.get(**gc_kwargs)
+ except Exception, msg:
+ raise LayerMapError('Geometry column does not exist for model.
(did you run syncdb?):\n %s' % msg)
- if isinstance(fld, OFTString) and self.encoding:
- # The encoding for OGR data sources may be specified
here
- # (e.g., 'cp437' for Census Bureau boundary files).
- val = unicode(fld.value, self.encoding)
- else:
- val = fld.value
+ def make_multi(self, geom_name, model_type):
+ "Determines whether the geometry should be made into a
GeometryCollection."
+ return (geom_name in self.MULTI_TYPES) and
(model_type.startswith('Multi'))
- if is_fk:
- rel_obj = None
- field_name = model_field[:-3]
- try:
- #FIXME: refactor to efficiently fetch FKs.
- # Requires significant re-work. :-/
- rel = self.model._meta.get_field(field_name).rel
- rel_obj = rel.to._default_manager.get(**{('%s__exact'
% rel.field_name):val})
- except ObjectDoesNotExist:
- all_prepped = False
-
- kwargs[model_field[:-3]] = rel_obj
- else:
- kwargs[model_field] = val
+ def map_foreign_key(self, django_field):
+ "Handles fields within foreign keys for the given field."
+ if not django_field.__class__ is ForeignKey:
+ # Returning the field's class name.
+ return django_field.__class__.__name__
+ else:
+ # Otherwise, getting the type of the related field's
+ # from the Foreign key.
+ rf = django_field.rel.get_related_field()
+ return rf.get_internal_type()
- # Constructing the model using the constructed keyword args
- if all_prepped:
- m = self.model(**kwargs)
+ def save(self, verbose=False):
+ "Runs the layer mapping on the given SHP file, and saves to the
database."
+
+ @self.transaction_decorator
+ def _save():
+ num_feat = 0
+ num_saved = 0
- # Saving the model
+ for feat in self.layer:
+ num_feat += 1
+ # Getting the keyword arguments
try:
+ kwargs, all_prepped = self.feature_kwargs(feat)
+ except LayerMapError, msg:
+ # Something borked the validation
+ if self.strict: raise
+ elif not self.silent:
+ print 'Ignoring Feature ID %s because: %s' %
(feat.fid, msg)
+ else:
+ # Constructing the model using the constructed keyword args
if all_prepped:
- m.save()
- if verbose: print 'Saved: %s' % str(m)
+ m = self.model(**kwargs)
+ try:
+ m.save()
+ num_saved += 1
+ if verbose: print 'Saved: %s' % m
+ except SystemExit:
+ raise
+ except Exception, msg:
+ if self.transaction_mode == 'autocommit':
+ # Rolling back the transaction so that other
model saves
+ # will work.
+ transaction.rollback_unless_managed()
+ if self.strict:
+ # Bailing out if the `strict` keyword is set.
+ if not self.silent:
+ print 'Failed to save the feature (id: %s)
into the model with the keyword arguments:' % feat.fid
+ print kwargs
+ raise
+ elif not self.silent:
+ print 'Failed to save %s:\n %s\nContinuing' %
(kwargs, msg)
else:
- print "Skipping %s due to missing relation." % kwargs
- except SystemExit:
- raise
- except Exception, e:
- print "Failed to save %s\n Continuing" % kwargs
+ print 'Skipping %s due to missing relation.' % kwargs
+
+ # Printing progress information, if requested.
+ if self.progress and num_feat % self.interval == 0:
+ print 'Processed %d features, saved %d ...' % (num_feat,
num_saved)
+
+ # Calling our defined function, which will use the specified
+ # trasaction mode.
+ _save()
Modified: django/branches/gis/django/contrib/gis/utils/ogrinfo.py
===================================================================
--- django/branches/gis/django/contrib/gis/utils/ogrinfo.py 2007-11-17
21:38:36 UTC (rev 6686)
+++ django/branches/gis/django/contrib/gis/utils/ogrinfo.py 2007-11-17
21:57:12 UTC (rev 6687)
@@ -33,12 +33,12 @@
width = max(*map(len,layer.fields))
fmt = " %%%ss: %%s" % width
- for i, feature in enumerate(layer[:num_features]):
- print "=== Feature %s" % i
- for field in layer.fields:
- fld_typ = feature[field].__class__.__name__.replace('OFT', '')
- output = fmt % (field, fld_typ)
- val = feature.get(field)
+ for j, feature in enumerate(layer[:num_features]):
+ print "=== Feature %s" % j
+ for fld_name in layer.fields:
+ type_name = feature[fld_name].type_name
+ output = fmt % (fld_name, type_name)
+ val = feature.get(fld_name)
if val:
if isinstance(val, str):
val_fmt = ' ("%s")'
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"Django updates" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to [EMAIL PROTECTED]
For more options, visit this group at
http://groups.google.com/group/django-updates?hl=en
-~----------~----~----~----~------~----~------~--~---
