Oliver Gottwald ha scritto:
Hi,
I have a model with 120000+ points. It slows at load time to 15-40
seconds per 1000. I think that the issue is that I'm using the default
FeatureCollection.
Should I use a DataFeatureCollection? If so does anyone have an example
of implementation for DataFeatureCollection?
What FeatureCollection is best to use when loading large sets of data?
I'm using the below basic approach(extract from other simpler program)
but adding a lot of points:
FeatureCollection collection = FeatureCollections.newCollection();
GeometryFactory factory = JTSFactoryFinder.getGeometryFactory(null);
ArrayList coordList = new ArrayList();
Collection cl = allPoints.values();
Iterator it = cl.iterator();
while (it.hasNext()){
XyzPoint xyz = (XyzPoint)it.next();
coordList.add(new Coordinate(xyz.getX(),xyz.getY()));
//featureBuilder.set("name",xyz.getName());
}
Coordinate[] coords = CoordinateArrays.toCoordinateArray(coordList);
LineString lineString = factory.createLineString(coords);
//featureBuilder.set("name", "testing123");
featureBuilder.add(lineString);
collection.add(featureBuilder.buildFeature(null));
The default feature collection and the memory data stores are really
bad when it comes to performance and scalability, they were
though only as "demos" and who made them cared only to make them
correct: reading straight from shapefile is normally faster than
accessing those things in memory.
I don't think we have an in memory feature collection with good
performance, as everybody stores their data somewhere, files,
databases, and just uses the specific datastores to handle
them.
If I were in your position I would make a custom datastore
that builds features out of whatever storage your
model is contained in but... I have experience in dong that.
A less work alternative is to make your own in memory collection.
Or better, make yourself an in memory feature source, so that
you can still do something smart to fast filter out
data.
You can take the attached feature source and adapt it to your
needs. It was made to be a simple full in memory cache for
rendering and wraps another feature source, but with little
adaptations you can make it work by just taking a list of
features as an input instead.
It will spatially index your data making retrieval by bbox
quite a bit faster (when you are zoomed in).
If you can turn that into something useful plase share it
back :-)
Cheers
Andrea
--
Andrea Aime
OpenGeo - http://opengeo.org
Expert service straight from the developers.
package org.geotools.demo;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.geotools.data.DataSourceException;
import org.geotools.data.DataStore;
import org.geotools.data.DefaultQuery;
import org.geotools.data.FeatureListener;
import org.geotools.data.FeatureSource;
import org.geotools.data.Query;
import org.geotools.data.QueryCapabilities;
import org.geotools.data.ResourceInfo;
import org.geotools.data.store.FilteringIterator;
import org.geotools.data.store.ReTypingIterator;
import org.geotools.factory.CommonFactoryFinder;
import org.geotools.factory.Hints;
import org.geotools.feature.DefaultFeatureCollection;
import org.geotools.feature.FeatureCollection;
import org.geotools.feature.FeatureIterator;
import org.geotools.feature.collection.AbstractFeatureCollection;
import org.geotools.feature.simple.SimpleFeatureBuilder;
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
import org.geotools.geometry.jts.ReferencedEnvelope;
import org.opengis.feature.Feature;
import org.opengis.feature.IllegalAttributeException;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.AttributeDescriptor;
import org.opengis.feature.type.FeatureType;
import org.opengis.feature.type.Name;
import org.opengis.filter.And;
import org.opengis.filter.Filter;
import org.opengis.filter.FilterFactory;
import org.opengis.filter.expression.Expression;
import org.opengis.filter.expression.Literal;
import org.opengis.filter.expression.PropertyName;
import org.opengis.filter.spatial.BBOX;
import org.opengis.filter.spatial.BinarySpatialOperator;
import org.opengis.filter.spatial.Contains;
import org.opengis.filter.spatial.Crosses;
import org.opengis.filter.spatial.DWithin;
import org.opengis.filter.spatial.Equals;
import org.opengis.filter.spatial.Intersects;
import org.opengis.filter.spatial.Overlaps;
import org.opengis.filter.spatial.Touches;
import org.opengis.filter.spatial.Within;
import com.vividsolutions.jts.geom.Envelope;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.index.SpatialIndex;
import com.vividsolutions.jts.index.strtree.STRtree;
/**
* A caching feature source for fast data access.
*
*/
public class CachingFeatureSource implements FeatureSource {
private FeatureSource wrapped;
private SpatialIndex index;
private boolean dirty;
private Query cachedQuery;
private Envelope cachedBounds;
private SimpleFeatureType cachedSchema;
private Envelope originalBounds;
private static FilterFactory ff = CommonFactoryFinder.getFilterFactory(null);
private static final Set<Class> supportedFilterTypes = new HashSet<Class>(Arrays.asList(
BBOX.class, Contains.class, Crosses.class, DWithin.class, Equals.class,
Intersects.class, Overlaps.class, Touches.class, Within.class));
public CachingFeatureSource(FeatureSource original) throws IOException {
this.wrapped = original;
this.originalBounds = original.getBounds();
if (originalBounds == null)
originalBounds = new Envelope(-Double.MAX_VALUE, Double.MAX_VALUE, -Double.MAX_VALUE,
Double.MAX_VALUE);
}
private void fillCache(Query query) throws IOException {
System.out.println("Refilling cache from " + query);
Query cloned = new DefaultQuery(query);
cloned.getHints().remove(Hints.GEOMETRY_DISTANCE);
FeatureCollection features = wrapped.getFeatures(cloned);
FeatureIterator fi = features.features();
index = null;
STRtree newIndex = new STRtree();
while (fi.hasNext()) {
// consider turning all geometries into packed ones, to save space
Feature f = fi.next();
newIndex.insert(ReferencedEnvelope.reference(f.getBounds()), f);
}
fi.close();
index = newIndex;
cachedQuery = query;
cachedSchema = (SimpleFeatureType) features.getSchema();
cachedBounds = getEnvelope(query.getFilter());
dirty = false;
}
public void addFeatureListener(FeatureListener listener) {
wrapped.addFeatureListener(listener);
}
public void removeFeatureListener(FeatureListener listener) {
wrapped.removeFeatureListener(listener);
}
public DataStore getDataStore() {
return (DataStore) wrapped.getDataStore();
}
public ReferencedEnvelope getBounds() throws IOException {
return wrapped.getBounds();
}
public ReferencedEnvelope getBounds(Query query) throws IOException {
return wrapped.getBounds(query);
}
public int getCount(Query query) throws IOException {
return wrapped.getCount(query);
}
public FeatureType getSchema() {
return wrapped.getSchema();
}
public FeatureCollection getFeatures() throws IOException {
return getFeatures(Filter.INCLUDE);
}
public FeatureCollection getFeatures(Filter filter) throws IOException {
return getFeatures(new DefaultQuery(wrapped.getSchema().getName().getLocalPart(), filter));
}
public FeatureCollection getFeatures(Query query) throws IOException {
String schemaName = wrapped.getSchema().getName().getLocalPart();
if (query.getTypeName() != null && !schemaName.equals(query.getTypeName())) {
throw new DataSourceException("Typename mismatch, query asks for '"
+ query.getTypeName() + " but this feature source provides '" + schemaName
+ "'");
}
if (index == null || dirty || !isSubQuery(query)) {
fillCache(query);
}
return getFeatureCollection(query, getEnvelope(query.getFilter()));
}
private FeatureCollection getFeatureCollection(Query query, Envelope bounds) throws IOException {
try {
SimpleFeatureType alternate = cachedSchema;
if (query.getPropertyNames() != Query.ALL_NAMES) {
alternate = SimpleFeatureTypeBuilder.retype(cachedSchema, query.getPropertyNames());
if (alternate.equals(cachedSchema))
alternate = cachedSchema;
}
Filter f ;
if (query.getFilter() != null && query.getFilter().equals(Filter.EXCLUDE)) {
f = null;
} else {
f = query.getFilter();
}
List featureList = index.query(bounds);
return new CachingFeatureCollection(featureList, cachedSchema, alternate, f);
} catch (Exception e) {
throw new DataSourceException(
"Error occurred extracting features from the spatial index", e);
}
}
/**
* Same as DataUtilities.reType, but without the cloning that uselessly wastes CPU cycles...
*
* @param featureType
* @param feature
* @return
* @throws IllegalAttributeException
*/
public static SimpleFeature reType(SimpleFeatureType featureType, SimpleFeature feature)
throws IllegalAttributeException {
FeatureType origional = feature.getFeatureType();
if (featureType.equals(origional)) {
return SimpleFeatureBuilder.copy(feature);
}
String id = feature.getID();
int numAtts = featureType.getAttributeCount();
Object[] attributes = new Object[numAtts];
String xpath;
for (int i = 0; i < numAtts; i++) {
AttributeDescriptor curAttType = featureType.getDescriptor(i);
attributes[i] = feature.getAttribute(curAttType.getLocalName());
}
return SimpleFeatureBuilder.build(featureType, attributes, id);
}
boolean isSubQuery(Query query) {
// no cached data?
if (cachedQuery == null)
return false;
// do we miss some properties?
String[] cachedPropNames = cachedQuery.getPropertyNames();
String[] propNames = query.getPropertyNames();
if (cachedPropNames != Query.ALL_NAMES
&& (propNames == Query.ALL_NAMES || !Arrays.asList(cachedPropNames).containsAll(
Arrays.asList(propNames))))
return false;
Filter[] filters = splitFilters(query);
Filter[] cachedFilters = splitFilters(cachedQuery);
if (!filters[0].equals(cachedFilters[0]))
return false;
Envelope envelope = getEnvelope(filters[1]);
return cachedBounds.contains(envelope);
}
Envelope getEnvelope(Filter filter) {
Envelope result = originalBounds;
if (filter instanceof And) {
Envelope bounds = new Envelope();
for (Iterator iter = ((And) filter).getChildren().iterator(); iter.hasNext();) {
Filter f = (Filter) iter.next();
Envelope e = getEnvelope(f);
if (e == null)
return null;
else
bounds.expandToInclude(e);
}
result = bounds;
} else if (filter instanceof BinarySpatialOperator) {
BinarySpatialOperator gf = (BinarySpatialOperator) filter;
if (supportedFilterTypes.contains(gf.getClass())) {
Expression lg = gf.getExpression1();
Expression rg = gf.getExpression2();
if (lg instanceof Literal) {
Geometry g = (Geometry) ((Literal) lg).getValue();
if (rg instanceof PropertyName)
result = g.getEnvelopeInternal();
} else if (rg instanceof Literal) {
Geometry g = (Geometry) ((Literal) rg).getValue();
if (lg instanceof PropertyName)
result = g.getEnvelopeInternal();
}
}
}
return result.intersection(originalBounds);
}
/**
* Splits a query into two parts, a spatial component that can be turned into a bbox filter (by
* including some more feature in the result) and a residual component that we cannot address
* with the spatial index
*
* @param query
*/
Filter[] splitFilters(Query query) {
Filter filter = query.getFilter();
if (filter == null || filter.equals(Filter.EXCLUDE)) {
return new Filter[] { Filter.EXCLUDE, bboxFilter(originalBounds) };
}
if (!(filter instanceof And)) {
Envelope envelope = getEnvelope(filter);
if (envelope == null)
return new Filter[] { Filter.EXCLUDE, bboxFilter(originalBounds) };
else
return new Filter[] { Filter.EXCLUDE, bboxFilter(envelope) };
}
And and = (And) filter;
List residuals = new ArrayList();
List bboxBacked = new ArrayList();
for (Iterator it = and.getChildren().iterator(); it.hasNext();) {
Filter child = (Filter) it.next();
if (getEnvelope(child) != null) {
bboxBacked.add(child);
} else {
residuals.add(child);
}
}
return new Filter[] { (Filter) ff.and(residuals), (Filter) ff.and(bboxBacked) };
}
private BBOX bboxFilter(Envelope bbox) {
// GeometryFilterImpl gf = (GeometryFilterImpl) ff
// .createGeometryFilter(GeometryFilter.GEOMETRY_BBOX);
// gf.setExpression1(ff.createAttributeExpression(wrapped.getSchema().getDefaultGeometry()));
// gf.setExpression2(ff.createBBoxExpression(bbox));
// return gf;
return ff.bbox(wrapped.getSchema().getGeometryDescriptor().getLocalName(), bbox.getMinX(),
bbox.getMinY(), bbox.getMaxX(), bbox.getMaxY(), null);
}
public ResourceInfo getInfo() {
return wrapped.getInfo();
}
public Name getName() {
return wrapped.getName();
}
public QueryCapabilities getQueryCapabilities() {
return wrapped.getQueryCapabilities();
}
public Set getSupportedHints() {
HashSet hints = new HashSet(wrapped.getSupportedHints());
hints.remove(Hints.FEATURE_DETACHED);
return hints;
}
/**
* A custom feature collection to avoid the {...@link DefaultFeatureCollection} nasty overhead
* @author aaime
*
*/
static final class CachingFeatureCollection extends AbstractFeatureCollection {
private List<SimpleFeature> features;
private SimpleFeatureType sourceSchema;
private SimpleFeatureType targetSchema;
private Filter filter;
protected CachingFeatureCollection(List<SimpleFeature> features, SimpleFeatureType sourceSchema,
SimpleFeatureType targetSchema, Filter filter) {
super(targetSchema);
this.features = features;
this.sourceSchema = sourceSchema;
this.targetSchema = targetSchema;
this.filter = filter;
}
@Override
public int size() {
return features.size();
}
@Override
protected Iterator openIterator() {
Iterator it = features.iterator();
if(filter != null) {
it = new FilteringIterator<Feature>(it, filter);
}
if(targetSchema != sourceSchema) {
it = new ReTypingIterator(it, sourceSchema, targetSchema);
}
return it;
}
@Override
protected void closeIterator(Iterator close) {
// nothing to do there
}
}
}------------------------------------------------------------------------------
Download Intel® Parallel Studio Eval
Try the new software tools for yourself. Speed compiling, find bugs
proactively, and fine-tune applications for parallel performance.
See why Intel Parallel Studio got high marks during beta.
http://p.sf.net/sfu/intel-sw-dev
_______________________________________________
Geotools-gt2-users mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/geotools-gt2-users
