This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch 590-flatgeobuf in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 7ead5ad4a15c72dacf6c757c60a8ddab74c32de7 Author: Bertil Chapuis <[email protected]> AuthorDate: Sat Feb 25 14:59:05 2023 +0100 Replace feature by dataframe --- .../AbstractDataCollection.java} | 25 ++- .../baremaps/collection/AppendOnlyBuffer.java | 2 +- .../apache/baremaps/collection/DataCollection.java | 13 +- .../collection/DataCollectionException.java | 8 +- .../org/apache/baremaps/collection/DataList.java | 2 +- .../AbstractDataFrame.java} | 10 +- .../Resource.java => dataframe/Column.java} | 6 +- .../Aggregate.java => dataframe/ColumnImpl.java} | 5 +- .../FeatureSet.java => dataframe/DataFrame.java} | 10 +- .../DataFrameException.java} | 22 +-- .../DataFrameImpl.java} | 25 +-- .../DataStore.java} | 13 +- .../FeatureSet.java => dataframe/DataType.java} | 10 +- .../DataTypeImpl.java} | 17 +- .../ReadableAggregate.java => dataframe/Row.java} | 13 +- .../org/apache/baremaps/dataframe/RowImpl.java | 40 ++++ .../org/apache/baremaps/feature/FeatureImpl.java | 55 ------ .../apache/baremaps/feature/FeatureTypeImpl.java | 51 ------ .../baremaps/feature/ReadableFeatureSet.java | 24 --- .../baremaps/feature/ReadableFeatureStream.java | 37 ---- .../apache/baremaps/feature/WritableAggregate.java | 25 --- .../baremaps/feature/WritableFeatureSet.java | 24 --- .../baremaps/mvt/expression/Expressions.java | 110 +++++------ .../storage/DataFrameProjectionTransform.java | 63 +++++++ .../storage/FeatureSetProjectionTransform.java | 58 ------ .../storage/flatgeobuf/FeatureIterator.java | 21 +-- .../storage/flatgeobuf/FlatGeoBufFeatureSet.java | 202 ++++++++++----------- ...FeatureConversions.java => RowConversions.java} | 77 ++++---- .../storage/geopackage/GeoPackageDatabase.java | 39 +++- .../storage/geopackage/GeoPackageTable.java | 62 +++---- .../{PostgresTable.java => PostgresDataFrame.java} | 31 ++-- .../storage/postgres/PostgresDatabase.java | 143 ++++++++------- .../storage/shapefile/ShapefileDataFrame.java | 102 +++++++++++ .../storage/shapefile/ShapefileDirectory.java | 40 +++- .../storage/shapefile/ShapefileFeatureSet.java | 92 ---------- .../shapefile/internal/DbaseByteReader.java | 8 +- .../shapefile/internal/InputFeatureStream.java | 24 +-- .../shapefile/internal/ShapefileByteReader.java | 40 ++-- .../shapefile/internal/ShapefileReader.java | 10 +- .../org/apache/baremaps/workflow/tasks/Entity.java | 36 ++-- .../baremaps/workflow/tasks/ImportGeoPackage.java | 13 +- .../baremaps/workflow/tasks/ImportShapefile.java | 8 +- .../workflow/tasks/TransformEntityCollection.java | 55 ++++-- .../org/apache/baremaps/mvt/ExpressionsTest.java | 28 +-- .../flatgeobuf/FlatGeoBufFeatureSetTest.java | 7 +- 45 files changed, 797 insertions(+), 909 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/Feature.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/AbstractDataCollection.java similarity index 55% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/Feature.java rename to baremaps-core/src/main/java/org/apache/baremaps/collection/AbstractDataCollection.java index 93e6d8db..0b0d41a4 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/Feature.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/AbstractDataCollection.java @@ -10,20 +10,27 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.collection; -import java.util.Map; +import java.util.AbstractCollection; -public interface Feature { - - FeatureType getType(); - - void setProperty(String name, Object value); +/** + * An abstract collection of data elements that can hold a large number of elements. + * + * @param <E> The type of the data. + */ +public abstract class AbstractDataCollection<E> extends AbstractCollection<E> + implements DataCollection<E> { - Object getProperty(String name); - Map<String, Object> getProperties(); + /** + * {@inheritDoc} + */ + @Override + public int size() { + return (int) Math.min(sizeAsLong(), Integer.MAX_VALUE); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java index 1e582daa..e46d9a95 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/AppendOnlyBuffer.java @@ -31,7 +31,7 @@ import org.apache.baremaps.collection.type.DataType; * * @param <E> The type of the data. */ -public class AppendOnlyBuffer<E> extends DataCollection<E> { +public class AppendOnlyBuffer<E> extends AbstractDataCollection<E> { private final DataType<E> dataType; private final Memory memory; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollection.java index 8a9d9831..394af1ae 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollection.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollection.java @@ -12,16 +12,9 @@ package org.apache.baremaps.collection; +import java.util.Collection; - -import java.util.AbstractCollection; - -/** - * A abstract collection of data elements that can hold a large number of elements. - * - * @param <E> The type of the data. - */ -public abstract class DataCollection<E> extends AbstractCollection<E> { +public interface DataCollection<E> extends Collection<E> { /** * Returns the number of values stored in the data store. @@ -31,7 +24,7 @@ public abstract class DataCollection<E> extends AbstractCollection<E> { public abstract long sizeAsLong(); /** {@inheritDoc} */ - public int size() { + default int size() { return (int) Math.min(sizeAsLong(), Integer.MAX_VALUE); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java index c2613527..d19d4e2c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java @@ -14,11 +14,11 @@ package org.apache.baremaps.collection; /** Signals that an exception occurred in a store. */ public class DataCollectionException extends RuntimeException { - /** Constructs a {@code BlobStoreException} with {@code null} as its error detail message. */ + /** Constructs a {@code DataCollectionException} with {@code null} as its error detail message. */ public DataCollectionException() {} /** - * Constructs an {@code StoreException} with the specified detail message. + * Constructs an {@code DataCollectionException} with the specified detail message. * * @param message the message */ @@ -27,7 +27,7 @@ public class DataCollectionException extends RuntimeException { } /** - * Constructs a {@code StoreException} with the specified cause. + * Constructs a {@code DataCollectionException} with the specified cause. * * @param cause the cause */ @@ -36,7 +36,7 @@ public class DataCollectionException extends RuntimeException { } /** - * Constructs a {@code StoreException} with the specified detail message and cause. + * Constructs a {@code DataCollectionException} with the specified detail message and cause. * * @param message the message * @param cause the cause diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataList.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/DataList.java index a0c1aff7..98c732fc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataList.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/DataList.java @@ -22,7 +22,7 @@ import java.util.NoSuchElementException; * * @param <E> The type of the data. */ -public abstract class DataList<E> extends DataCollection<E> { +public abstract class DataList<E> extends AbstractDataCollection<E> { /** * Appends a value to the list and returns its index. diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/AbstractDataFrame.java similarity index 73% copy from baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java copy to baremaps-core/src/main/java/org/apache/baremaps/dataframe/AbstractDataFrame.java index 2dcc4534..8a0349a4 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/AbstractDataFrame.java @@ -10,14 +10,10 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; +import org.apache.baremaps.collection.AbstractDataCollection; - -import java.io.IOException; - -public interface FeatureSet extends Resource { - - FeatureType getType() throws IOException; +public abstract class AbstractDataFrame extends AbstractDataCollection<Row> implements DataFrame { } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/Resource.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/Column.java similarity index 85% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/Resource.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/Column.java index a9686585..174d86ad 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/Resource.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/Column.java @@ -10,10 +10,12 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; +public interface Column { + String name(); -public interface Resource { + Class type(); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/Aggregate.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/ColumnImpl.java similarity index 83% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/Aggregate.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/ColumnImpl.java index a13eb326..ad3928c7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/Aggregate.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/ColumnImpl.java @@ -10,7 +10,8 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; + +public record ColumnImpl(String name, Class type) implements Column { -public interface Aggregate extends Resource { } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrame.java similarity index 76% copy from baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java copy to baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrame.java index 2dcc4534..3efccf11 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrame.java @@ -10,14 +10,12 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; +import org.apache.baremaps.collection.DataCollection; +public interface DataFrame extends DataCollection<Row> { -import java.io.IOException; - -public interface FeatureSet extends Resource { - - FeatureType getType() throws IOException; + DataType dataType(); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrameException.java similarity index 55% copy from baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java copy to baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrameException.java index c2613527..1a2e746b 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/collection/DataCollectionException.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrameException.java @@ -10,38 +10,38 @@ * the License. */ -package org.apache.baremaps.collection; +package org.apache.baremaps.dataframe; -/** Signals that an exception occurred in a store. */ -public class DataCollectionException extends RuntimeException { - /** Constructs a {@code BlobStoreException} with {@code null} as its error detail message. */ - public DataCollectionException() {} +/** Signals that an exception occurred in a data frame. */ +public class DataFrameException extends RuntimeException { + /** Constructs a {@code DataFrameException} with {@code null} as its error detail message. */ + public DataFrameException() {} /** - * Constructs an {@code StoreException} with the specified detail message. + * Constructs an {@code DataFrameException} with the specified detail message. * * @param message the message */ - public DataCollectionException(String message) { + public DataFrameException(String message) { super(message); } /** - * Constructs a {@code StoreException} with the specified cause. + * Constructs a {@code DataFrameException} with the specified cause. * * @param cause the cause */ - public DataCollectionException(Throwable cause) { + public DataFrameException(Throwable cause) { super(cause); } /** - * Constructs a {@code StoreException} with the specified detail message and cause. + * Constructs a {@code DataFrameException} with the specified detail message and cause. * * @param message the message * @param cause the cause */ - public DataCollectionException(String message, Throwable cause) { + public DataFrameException(String message, Throwable cause) { super(message, cause); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/PropertyType.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrameImpl.java similarity index 60% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/PropertyType.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrameImpl.java index 634657c4..4b0ffd27 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/PropertyType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataFrameImpl.java @@ -10,24 +10,25 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; -public class PropertyType<T> { +import java.util.ArrayList; - private final String name; - private final Class<T> type; +public class DataFrameImpl extends ArrayList<Row> implements DataFrame { - public PropertyType(String name, Class<T> type) { - this.name = name; - this.type = type; - } + private final DataType dataType; - public String getName() { - return name; + public DataFrameImpl(DataType dataType) { + this.dataType = dataType; } - public Class<T> getType() { - return type; + @Override + public long sizeAsLong() { + return size(); } + @Override + public DataType dataType() { + return dataType; + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataStore.java similarity index 67% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureCollection.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataStore.java index 5a4d8eba..ce89b957 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureCollection.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataStore.java @@ -10,15 +10,18 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; +import java.util.Collection; +public interface DataStore { -import java.io.IOException; -import java.util.Collection; + Collection<DataFrame> list() throws DataFrameException; + + DataFrame get(String name) throws DataFrameException; -public interface FeatureCollection extends Collection<Feature> { + void add(DataFrame value) throws DataFrameException; - FeatureType getType() throws IOException; + void remove(String name) throws DataFrameException; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataType.java similarity index 78% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataType.java index 2dcc4534..29352566 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureSet.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataType.java @@ -10,14 +10,16 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; +import java.util.List; +public interface DataType { -import java.io.IOException; + String name(); -public interface FeatureSet extends Resource { + List<Column> columns(); - FeatureType getType() throws IOException; + Row newInstance(); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataTypeImpl.java similarity index 66% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataTypeImpl.java index 52c07fe0..ee362fd2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureType.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/DataTypeImpl.java @@ -10,16 +10,15 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; -public interface FeatureType { - String getName(); +public record DataTypeImpl(String name, List<Column> columns) implements DataType { - Map<String, PropertyType> getPropertyTypes(); - - PropertyType getPropertyType(String name); - - Feature newInstance(); + @Override + public Row newInstance() { + return new RowImpl(this, new ArrayList(columns.size())); + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableAggregate.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/Row.java similarity index 73% rename from baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableAggregate.java rename to baremaps-core/src/main/java/org/apache/baremaps/dataframe/Row.java index 2493c211..8b291119 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableAggregate.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/Row.java @@ -10,15 +10,18 @@ * the License. */ -package org.apache.baremaps.feature; +package org.apache.baremaps.dataframe; +import java.util.List; +public interface Row { -import java.io.IOException; -import java.util.stream.Stream; + DataType dataType(); -public interface ReadableAggregate extends Aggregate { + List<Object> values(); - Stream<Resource> read() throws IOException; + Object get(String name); + + void set(String name, Object value); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/dataframe/RowImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/RowImpl.java new file mode 100644 index 00000000..027b383e --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/dataframe/RowImpl.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.dataframe; + +import java.util.List; + +public record RowImpl(DataType dataType, List values) implements Row { + + @Override + public Object get(String name) { + for (int i = 0; i < dataType().columns().size(); i++) { + if (dataType().columns().get(i).name().equals(name)) { + return values.get(i); + } + } + throw new IllegalArgumentException("Column " + name + " not found."); + } + + @Override + public void set(String name, Object value) { + for (int i = 0; i < dataType().columns().size(); i++) { + if (dataType().columns().get(i).name().equals(name)) { + values.set(i, value); + return; + } + } + throw new IllegalArgumentException("Column " + name + " not found."); + } + +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java deleted file mode 100644 index 8e0fc0c2..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - - - -import com.google.common.collect.ImmutableMap; -import java.util.HashMap; -import java.util.Map; - -public final class FeatureImpl implements Feature { - - private FeatureType type; - - private Map<String, Object> properties; - - public FeatureImpl(FeatureType type) { - this(type, new HashMap<>()); - } - - public FeatureImpl(FeatureType type, Map<String, Object> properties) { - this.type = type; - this.properties = properties; - } - - @Override - public FeatureType getType() { - return type; - } - - @Override - public void setProperty(String name, Object value) { - properties.put(name, value); - } - - @Override - public Object getProperty(String name) { - return properties.get(name); - } - - public Map<String, Object> getProperties() { - return ImmutableMap.copyOf(properties); - } - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureTypeImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureTypeImpl.java deleted file mode 100644 index 1cda4086..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/FeatureTypeImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - - - -import com.google.common.collect.ImmutableMap; -import java.util.Map; - -public class FeatureTypeImpl implements FeatureType { - - private final String name; - - private final Map<String, PropertyType> propertyTypes; - - public FeatureTypeImpl(String name, Map<String, PropertyType> propertyTypes) { - this.name = name; - this.propertyTypes = propertyTypes; - } - - @Override - public String getName() { - return name; - } - - @Override - public Map<String, PropertyType> getPropertyTypes() { - return ImmutableMap.copyOf(propertyTypes); - } - - @Override - public PropertyType getPropertyType(String name) { - return propertyTypes.get(name); - } - - @Override - public Feature newInstance() { - return new FeatureImpl(this); - } - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableFeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableFeatureSet.java deleted file mode 100644 index 8fa615b4..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableFeatureSet.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - - - -import java.io.IOException; -import java.util.stream.Stream; - -public interface ReadableFeatureSet extends FeatureSet { - - Stream<? extends Feature> read() throws IOException; - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableFeatureStream.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableFeatureStream.java deleted file mode 100644 index e1969fae..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/ReadableFeatureStream.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - -import java.util.stream.Stream; - -public class ReadableFeatureStream implements ReadableFeatureSet { - - private final FeatureType featureType; - - private final Stream<? extends Feature> stream; - - public ReadableFeatureStream(FeatureType featureType, Stream<? extends Feature> stream) { - this.featureType = featureType; - this.stream = stream; - } - - @Override - public FeatureType getType() { - return featureType; - } - - @Override - public Stream<? extends Feature> read() { - return stream; - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/WritableAggregate.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/WritableAggregate.java deleted file mode 100644 index 87d300c4..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/WritableAggregate.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - - - -import java.io.IOException; - -public interface WritableAggregate extends Aggregate { - - void write(Resource resource) throws IOException; - - void remove(Resource resource) throws IOException; - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/feature/WritableFeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/feature/WritableFeatureSet.java deleted file mode 100644 index a7541ef4..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/feature/WritableFeatureSet.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.feature; - - - -import java.io.IOException; -import java.util.stream.Stream; - -public interface WritableFeatureSet extends FeatureSet { - - void write(Stream<Feature> features) throws IOException; - -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java b/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java index f421dd77..1cd7d3fc 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/mvt/expression/Expressions.java @@ -32,7 +32,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; -import org.apache.baremaps.feature.Feature; +import org.apache.baremaps.dataframe.Row; import org.locationtech.jts.geom.*; public interface Expressions { @@ -41,7 +41,7 @@ public interface Expressions { String name(); - T evaluate(Feature feature); + T evaluate(Row feature); } @@ -53,7 +53,7 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { + public Object evaluate(Row row) { return value; } } @@ -66,8 +66,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - Object value = expression.evaluate(feature); + public Object evaluate(Row row) { + Object value = expression.evaluate(row); if (value instanceof List list && index >= 0 && index < list.size()) { return list.get(index); } @@ -83,8 +83,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return feature.getProperty(property); + public Object evaluate(Row row) { + return row.get(property); } } @@ -96,8 +96,8 @@ public interface Expressions { } @Override - public Boolean evaluate(Feature feature) { - return feature.getProperty(property) != null; + public Boolean evaluate(Row row) { + return row.get(property) != null; } } @@ -109,8 +109,8 @@ public interface Expressions { } @Override - public Boolean evaluate(Feature feature) { - var expressionValue = expression.evaluate(feature); + public Boolean evaluate(Row row) { + var expressionValue = expression.evaluate(row); if (expressionValue instanceof List list) { return list.contains(value); } else if (expressionValue instanceof String string) { @@ -129,8 +129,8 @@ public interface Expressions { } @Override - public Integer evaluate(Feature feature) { - var expressionValue = expression.evaluate(feature); + public Integer evaluate(Row row) { + var expressionValue = expression.evaluate(row); if (expressionValue instanceof List list) { return list.indexOf(value); } else if (expressionValue instanceof String string) { @@ -149,8 +149,8 @@ public interface Expressions { } @Override - public Integer evaluate(Feature feature) { - Object value = expression.evaluate(feature); + public Integer evaluate(Row row) { + Object value = expression.evaluate(row); if (value instanceof String string) { return string.length(); } else if (value instanceof List list) { @@ -173,14 +173,14 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - Object value = expression.evaluate(feature); - var startIndex = (Integer) start.evaluate(feature); + public Object evaluate(Row row) { + Object value = expression.evaluate(row); + var startIndex = (Integer) start.evaluate(row); if (value instanceof String string) { - var endIndex = end == null ? string.length() : (Integer) end.evaluate(feature); + var endIndex = end == null ? string.length() : (Integer) end.evaluate(row); return string.substring(startIndex, endIndex); } else if (value instanceof List list) { - var endIndex = end == null ? list.size() : (Integer) end.evaluate(feature); + var endIndex = end == null ? list.size() : (Integer) end.evaluate(row); return list.subList(startIndex, endIndex); } else { return List.of(); @@ -196,8 +196,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return !(boolean) expression.evaluate(feature); + public Object evaluate(Row row) { + return !(boolean) expression.evaluate(row); } } @@ -209,8 +209,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return new Not(new Equal(left, right)).evaluate(feature); + public Object evaluate(Row row) { + return new Not(new Equal(left, right)).evaluate(row); } } @@ -222,8 +222,8 @@ public interface Expressions { } @Override - public Boolean evaluate(Feature feature) { - return (double) left.evaluate(feature) < (double) right.evaluate(feature); + public Boolean evaluate(Row row) { + return (double) left.evaluate(row) < (double) right.evaluate(row); } } @@ -235,8 +235,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return (double) left.evaluate(feature) <= (double) right.evaluate(feature); + public Object evaluate(Row row) { + return (double) left.evaluate(row) <= (double) right.evaluate(row); } } @@ -248,8 +248,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return left.evaluate(feature).equals(right.evaluate(feature)); + public Object evaluate(Row row) { + return left.evaluate(row).equals(right.evaluate(row)); } } @@ -261,8 +261,8 @@ public interface Expressions { } @Override - public Boolean evaluate(Feature feature) { - return (double) left.evaluate(feature) > (double) right.evaluate(feature); + public Boolean evaluate(Row row) { + return (double) left.evaluate(row) > (double) right.evaluate(row); } } @@ -274,8 +274,8 @@ public interface Expressions { } @Override - public Boolean evaluate(Feature feature) { - return (double) left.evaluate(feature) >= (double) right.evaluate(feature); + public Boolean evaluate(Row row) { + return (double) left.evaluate(row) >= (double) right.evaluate(row); } } @@ -287,8 +287,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return expressions.stream().allMatch(expression -> (boolean) expression.evaluate(feature)); + public Object evaluate(Row row) { + return expressions.stream().allMatch(expression -> (boolean) expression.evaluate(row)); } } @@ -300,8 +300,8 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - return expressions.stream().anyMatch(expression -> (boolean) expression.evaluate(feature)); + public Object evaluate(Row row) { + return expressions.stream().anyMatch(expression -> (boolean) expression.evaluate(row)); } } @@ -313,11 +313,11 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { - if ((boolean) condition.evaluate(feature)) { - return then.evaluate(feature); + public Object evaluate(Row row) { + if ((boolean) condition.evaluate(row)) { + return then.evaluate(row); } else { - return otherwise.evaluate(feature); + return otherwise.evaluate(row); } } } @@ -330,9 +330,9 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { + public Object evaluate(Row row) { for (Expression expression : expressions) { - Object value = expression.evaluate(feature); + Object value = expression.evaluate(row); if (value != null) { return value; } @@ -350,20 +350,20 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { + public Object evaluate(Row row) { if (cases.size() % 2 != 0) { throw new IllegalArgumentException( "match expression must have an even number of arguments"); } - var inputValue = input.evaluate(feature); + var inputValue = input.evaluate(row); for (int i = 0; i < cases.size(); i += 2) { Expression condition = cases.get(i); Expression then = cases.get(i + 1); - if (inputValue.equals(condition.evaluate(feature))) { - return then.evaluate(feature); + if (inputValue.equals(condition.evaluate(row))) { + return then.evaluate(row); } } - return fallback.evaluate(feature); + return fallback.evaluate(row); } } @@ -375,7 +375,7 @@ public interface Expressions { } @Override - public Object evaluate(Feature feature) { + public Object evaluate(Row row) { throw new UnsupportedOperationException("within expression is not supported"); } } @@ -388,8 +388,8 @@ public interface Expressions { } @Override - public String evaluate(Feature feature) { - Object property = feature.getProperty("geom"); + public String evaluate(Row row) { + Object property = row.get("geom"); if (property instanceof Point) { return "Point"; } else if (property instanceof LineString) { @@ -518,9 +518,9 @@ public interface Expressions { return mapper.writeValueAsString(expression); } - static Predicate<Feature> asPredicate(Expression expression) { - return feature -> { - var result = expression.evaluate(feature); + static Predicate<Row> asPredicate(Expression expression) { + return row -> { + var result = expression.evaluate(row); if (result instanceof Boolean booleanResult) { return booleanResult; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/DataFrameProjectionTransform.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/DataFrameProjectionTransform.java new file mode 100644 index 00000000..9074d45e --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/DataFrameProjectionTransform.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage; + + + +import java.util.Iterator; +import org.apache.baremaps.collection.AbstractDataCollection; +import org.apache.baremaps.dataframe.Column; +import org.apache.baremaps.dataframe.DataFrame; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; +import org.apache.baremaps.openstreetmap.utils.ProjectionTransformer; +import org.locationtech.jts.geom.Geometry; + +public class DataFrameProjectionTransform extends AbstractDataCollection<Row> implements DataFrame { + + private final DataFrame dataFrame; + + private final ProjectionTransformer projectionTransformer; + + public DataFrameProjectionTransform(DataFrame dataFrame, + ProjectionTransformer projectionTransformer) { + this.dataFrame = dataFrame; + this.projectionTransformer = projectionTransformer; + } + + @Override + public DataType dataType() { + return dataFrame.dataType(); + } + + public Row transformProjection(Row row) { + var columns = dataType().columns().stream() + .filter(column -> column.type().isInstance(Geometry.class)).toList(); + for (Column column : columns) { + var name = column.name(); + var geometry = (Geometry) row.get(name); + row.set(name, projectionTransformer.transform(geometry)); + } + return row; + } + + @Override + public Iterator<Row> iterator() { + return dataFrame.stream().map(this::transformProjection).iterator(); + } + + @Override + public long sizeAsLong() { + return 0; + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java deleted file mode 100644 index c02beb7c..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/FeatureSetProjectionTransform.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.storage; - - - -import java.io.IOException; -import java.util.stream.Stream; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.ReadableFeatureSet; -import org.apache.baremaps.openstreetmap.utils.ProjectionTransformer; -import org.locationtech.jts.geom.Geometry; - -public class FeatureSetProjectionTransform implements ReadableFeatureSet { - - private final ReadableFeatureSet featureSetReader; - - private final ProjectionTransformer projectionTransformer; - - public FeatureSetProjectionTransform(ReadableFeatureSet featureSetReader, - ProjectionTransformer projectionTransformer) { - this.featureSetReader = featureSetReader; - this.projectionTransformer = projectionTransformer; - } - - @Override - public FeatureType getType() throws IOException { - return featureSetReader.getType(); - } - - @Override - public Stream<Feature> read() throws IOException { - return featureSetReader.read().map(this::transformProjection); - } - - public Feature transformProjection(Feature feature) { - for (var property : feature.getType().getPropertyTypes().values()) { - var name = property.getName(); - var value = feature.getProperty(name); - if (value instanceof Geometry geometry) { - var projectedGeometry = projectionTransformer.transform(geometry); - feature.setProperty(name, projectedGeometry); - } - } - return feature; - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java index a5b5affd..cb1dab60 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureIterator.java @@ -18,17 +18,16 @@ import java.nio.ByteOrder; import java.nio.channels.SeekableByteChannel; import java.util.Iterator; import java.util.NoSuchElementException; -import java.util.Spliterator; -import java.util.function.Consumer; -import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; import org.wololo.flatgeobuf.HeaderMeta; import org.wololo.flatgeobuf.generated.Feature; -public class FeatureIterator implements Iterator<org.apache.baremaps.feature.Feature> { +public class FeatureIterator implements Iterator<Row> { private final HeaderMeta headerMeta; - private final FeatureType featureType; + private final DataType dataType; private final SeekableByteChannel channel; @@ -37,10 +36,10 @@ public class FeatureIterator implements Iterator<org.apache.baremaps.feature.Fea private long cursor = 0; public FeatureIterator(SeekableByteChannel channel, HeaderMeta headerMeta, - FeatureType featureType) { + DataType dataType) { this.channel = channel; this.headerMeta = headerMeta; - this.featureType = featureType; + this.dataType = dataType; this.buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); } @@ -50,21 +49,21 @@ public class FeatureIterator implements Iterator<org.apache.baremaps.feature.Fea } @Override - public org.apache.baremaps.feature.Feature next() { + public Row next() { try { channel.read(buffer); buffer.flip(); var featureSize = buffer.getInt(); - var feature = - FeatureConversions.asFeature(headerMeta, featureType, Feature.getRootAsFeature(buffer)); + var row = + RowConversions.asFeature(headerMeta, dataType, Feature.getRootAsFeature(buffer)); buffer.position(Integer.BYTES + featureSize); buffer.compact(); cursor++; - return feature; + return row; } catch (IOException e) { throw new NoSuchElementException(e); } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java index b9b5edc9..6d22a890 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSet.java @@ -13,7 +13,6 @@ package org.apache.baremaps.storage.flatgeobuf; import com.google.flatbuffers.FlatBufferBuilder; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -21,11 +20,9 @@ import java.nio.channels.Channels; import java.nio.channels.SeekableByteChannel; import java.util.Collection; import java.util.Iterator; -import java.util.Map.Entry; - -import org.apache.baremaps.collection.DataCollection; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.collection.AbstractDataCollection; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; import org.locationtech.jts.geom.Geometry; import org.wololo.flatgeobuf.Constants; import org.wololo.flatgeobuf.GeometryConversions; @@ -35,124 +32,125 @@ import org.wololo.flatgeobuf.generated.GeometryType; public class FlatGeoBufFeatureSet { - private final SeekableByteChannel channel; + private final SeekableByteChannel channel; - private HeaderMeta headerMeta; + private HeaderMeta headerMeta; - private FeatureType featureType; + private DataType dataType; - public FlatGeoBufFeatureSet(SeekableByteChannel channel) { - this.channel = channel; - } + public FlatGeoBufFeatureSet(SeekableByteChannel channel) { + this.channel = channel; + } - public FlatGeoBufFeatureSet(SeekableByteChannel channel, FeatureType featureType) { - this.channel = channel; - this.featureType = featureType; - } + public FlatGeoBufFeatureSet(SeekableByteChannel channel, DataType dataType) { + this.channel = channel; + this.dataType = dataType; + } - public FeatureType getType() throws IOException { - readHeaderMeta(); - return featureType; - } + public DataType getType() throws IOException { + readHeaderMeta(); + return dataType; + } - public Collection<Feature> read() throws IOException { - readHeaderMeta(); + public Collection<Row> read() throws IOException { + readHeaderMeta(); - return new DataCollection<>() { + return new AbstractDataCollection<>() { - @Override - public Iterator<Feature> iterator() { - try { - channel.position(headerMeta.offset); + @Override + public Iterator<Row> iterator() { + try { + channel.position(headerMeta.offset); - // skip the index - var indexSize = - (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); - channel.position(headerMeta.offset + indexSize); + // skip the index + var indexSize = + (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + channel.position(headerMeta.offset + indexSize); - // create the feature stream - return new FeatureIterator(channel, headerMeta, featureType); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + // create the feature stream + return new FeatureIterator(channel, headerMeta, dataType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public long sizeAsLong() { - return headerMeta.featuresCount; - } + @Override + public long sizeAsLong() { + return headerMeta.featuresCount; + } - }; - } + }; + } - public void readHeaderMeta() throws IOException { - channel.position(0); - var headerMetaBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); - channel.read(headerMetaBuffer); - headerMetaBuffer.flip(); - headerMeta = HeaderMeta.read(headerMetaBuffer); - featureType = FeatureConversions.asFeatureType(headerMeta); - } + public void readHeaderMeta() throws IOException { + channel.position(0); + var headerMetaBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + channel.read(headerMetaBuffer); + headerMetaBuffer.flip(); + headerMeta = HeaderMeta.read(headerMetaBuffer); + dataType = RowConversions.asFeatureType(headerMeta); + } - public void write(Collection<Feature> features) throws IOException { - channel.position(0); + public void write(Collection<Row> features) throws IOException { + channel.position(0); - var outputStream = Channels.newOutputStream(channel); + var outputStream = Channels.newOutputStream(channel); - outputStream.write(Constants.MAGIC_BYTES); + outputStream.write(Constants.MAGIC_BYTES); - var bufferBuilder = new FlatBufferBuilder(); + var bufferBuilder = new FlatBufferBuilder(); - var headerMeta = new HeaderMeta(); - headerMeta.geometryType = GeometryType.Unknown; - headerMeta.indexNodeSize = 16; - headerMeta.featuresCount = features instanceof DataCollection<Feature> c ? c.sizeAsLong() : features.size(); - headerMeta.name = featureType.getName(); - headerMeta.columns = FeatureConversions.asColumns(featureType.getPropertyTypes()); + var headerMeta = new HeaderMeta(); + headerMeta.geometryType = GeometryType.Unknown; + headerMeta.indexNodeSize = 16; + headerMeta.featuresCount = + features instanceof AbstractDataCollection<Row>c ? c.sizeAsLong() : features.size(); + headerMeta.name = dataType.name(); + headerMeta.columns = RowConversions.asColumns(dataType.columns()); - HeaderMeta.write(headerMeta, outputStream, bufferBuilder); + HeaderMeta.write(headerMeta, outputStream, bufferBuilder); - var indexSize = - (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); + var indexSize = + (int) PackedRTree.calcSize((int) headerMeta.featuresCount, headerMeta.indexNodeSize); - for (int i = 0; i < indexSize; i++) { - outputStream.write(0); - } + for (int i = 0; i < indexSize; i++) { + outputStream.write(0); + } - var iterator = features.iterator(); - while (iterator.hasNext()) { - var feature = iterator.next(); - - var featureBuilder = new FlatBufferBuilder(); - var geometryOffset = 0; - var propertiesOffset = 0; - var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); - var i = 0; - for (Entry<String, Object> entry : feature.getProperties().entrySet()) { - if (entry.getValue() instanceof Geometry) { - try { - var geometry = (Geometry) entry.getValue(); - geometryOffset = GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - var column = headerMeta.columns.get(i); - var value = feature.getProperty(column.name); - propertiesBuffer.putShort((short) i); - FeatureConversions.writeValue(propertiesBuffer, column, value); - i++; - } - } - propertiesBuffer.flip(); - propertiesOffset = org.wololo.flatgeobuf.generated.Feature.createPropertiesVector(featureBuilder, propertiesBuffer); - - var featureOffset = - org.wololo.flatgeobuf.generated.Feature.createFeature(featureBuilder, geometryOffset, propertiesOffset, 0); - - featureBuilder.finishSizePrefixed(featureOffset); - - channel.write(featureBuilder.dataBuffer()); + var iterator = features.iterator(); + while (iterator.hasNext()) { + var row = iterator.next(); + var featureBuilder = new FlatBufferBuilder(); + var geometryOffset = 0; + var propertiesOffset = 0; + var propertiesBuffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN); + var i = 0; + for (Object value : row.values()) { + if (value instanceof Geometry geometry) { + try { + geometryOffset = + GeometryConversions.serialize(featureBuilder, geometry, headerMeta.geometryType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + var column = headerMeta.columns.get(i); + propertiesBuffer.putShort((short) i); + RowConversions.writeValue(propertiesBuffer, column, value); + i++; } + } + propertiesBuffer.flip(); + propertiesOffset = org.wololo.flatgeobuf.generated.Feature + .createPropertiesVector(featureBuilder, propertiesBuffer); + + var featureOffset = + org.wololo.flatgeobuf.generated.Feature.createFeature(featureBuilder, geometryOffset, + propertiesOffset, 0); + + featureBuilder.finishSizePrefixed(featureOffset); + + channel.write(featureBuilder.dataBuffer()); } + } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureConversions.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowConversions.java similarity index 77% rename from baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureConversions.java rename to baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowConversions.java index 6e290804..53d08391 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FeatureConversions.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/RowConversions.java @@ -14,44 +14,40 @@ package org.apache.baremaps.storage.flatgeobuf; import com.google.flatbuffers.FlatBufferBuilder; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.FeatureTypeImpl; -import org.apache.baremaps.feature.PropertyType; -import org.wololo.flatgeobuf.ColumnMeta; -import org.wololo.flatgeobuf.GeometryConversions; -import org.wololo.flatgeobuf.HeaderMeta; -import org.wololo.flatgeobuf.generated.Column; -import org.wololo.flatgeobuf.generated.ColumnType; -import org.wololo.flatgeobuf.generated.Crs; -import org.wololo.flatgeobuf.generated.Header; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.nio.charset.StandardCharsets; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.baremaps.dataframe.*; +import org.wololo.flatgeobuf.ColumnMeta; +import org.wololo.flatgeobuf.GeometryConversions; +import org.wololo.flatgeobuf.HeaderMeta; +import org.wololo.flatgeobuf.generated.ColumnType; +import org.wololo.flatgeobuf.generated.Crs; +import org.wololo.flatgeobuf.generated.Feature; +import org.wololo.flatgeobuf.generated.Header; -public class FeatureConversions { +public class RowConversions { - public static FeatureType asFeatureType(HeaderMeta headerMeta) { + public static DataType asFeatureType(HeaderMeta headerMeta) { var name = headerMeta.name; - var properties = headerMeta.columns.stream().collect( - Collectors.toMap( - column -> column.name, - column -> new PropertyType(column.name, column.getBinding()))); - return new FeatureTypeImpl(name, properties); + var columns = headerMeta.columns.stream() + .map(column -> new ColumnImpl(column.name, column.getBinding())) + .map(Column.class::cast) + .toList(); + return new DataTypeImpl(name, columns); } - public static org.apache.baremaps.feature.Feature asFeature(HeaderMeta headerMeta, - FeatureType featureType, org.wololo.flatgeobuf.generated.Feature feature) { - var properties = new HashMap<String, Object>(); + public static RowImpl asFeature(HeaderMeta headerMeta, DataType dataType, Feature feature) { + var values = new ArrayList(); var geometryBuffer = feature.geometry(); var geometry = GeometryConversions.deserialize(geometryBuffer, geometryBuffer.type()); - properties.put("geometry", geometry); + values.add(geometry); if (feature.propertiesLength() > 0) { var propertiesBuffer = feature.propertiesAsByteBuffer(); @@ -59,25 +55,26 @@ public class FeatureConversions { var type = propertiesBuffer.getShort(); var column = headerMeta.columns.get(type); var value = readValue(propertiesBuffer, column); - properties.put(column.name, value); + values.add(value); } } - return new org.apache.baremaps.feature.FeatureImpl(featureType, properties); + return new RowImpl(dataType, values); } public static void writeHeaderMeta(HeaderMeta headerMeta, WritableByteChannel channel, - FlatBufferBuilder builder) throws IOException { + FlatBufferBuilder builder) throws IOException { int[] columnsArray = headerMeta.columns.stream().mapToInt(c -> { int nameOffset = builder.createString(c.name); int type = c.type; - return Column.createColumn(builder, nameOffset, type, 0, 0, c.width, c.precision, c.scale, c.nullable, c.unique, - c.primary_key, 0); + return org.wololo.flatgeobuf.generated.Column.createColumn(builder, nameOffset, type, 0, 0, + c.width, c.precision, c.scale, c.nullable, c.unique, + c.primary_key, 0); }).toArray(); int columnsOffset = Header.createColumnsVector(builder, columnsArray); int nameOffset = 0; - if (headerMeta.name!=null) { + if (headerMeta.name != null) { nameOffset = builder.createString(headerMeta.name); } int crsOffset = 0; @@ -89,11 +86,12 @@ public class FeatureConversions { int envelopeOffset = 0; if (headerMeta.envelope != null) { envelopeOffset = Header.createEnvelopeVector(builder, - new double[] { headerMeta.envelope.getMinX(), headerMeta.envelope.getMinY(), headerMeta.envelope.getMaxX(), headerMeta.envelope.getMaxY() }); + new double[] {headerMeta.envelope.getMinX(), headerMeta.envelope.getMinY(), + headerMeta.envelope.getMaxX(), headerMeta.envelope.getMaxY()}); } Header.startHeader(builder); Header.addGeometryType(builder, headerMeta.geometryType); - Header.addIndexNodeSize(builder, headerMeta.indexNodeSize ); + Header.addIndexNodeSize(builder, headerMeta.indexNodeSize); Header.addColumns(builder, columnsOffset); Header.addEnvelope(builder, envelopeOffset); Header.addName(builder, nameOffset); @@ -138,7 +136,8 @@ public class FeatureConversions { case ColumnType.Json -> writeJson(propertiesBuffer, value); case ColumnType.DateTime -> writeDateTime(propertiesBuffer, value); case ColumnType.Binary -> writeBinary(propertiesBuffer, value); - default -> {} + default -> { + } }; } @@ -189,19 +188,19 @@ public class FeatureConversions { Double.class, ColumnType.Double, String.class, ColumnType.String); - public static List<ColumnMeta> asColumns(Map<String, PropertyType> propertyTypes) { - return propertyTypes.values().stream() - .map(FeatureConversions::asColumn) + public static List<ColumnMeta> asColumns(List<Column> columns) { + return columns.stream() + .map(RowConversions::asColumn) .collect(Collectors.toList()); } - public static ColumnMeta asColumn(PropertyType propertyType) { - var type = types.get(propertyType.getType()); + public static ColumnMeta asColumn(Column column) { + var type = types.get(column.type()); if (type == null) { - throw new IllegalArgumentException("Unsupported type " + propertyType); + throw new IllegalArgumentException("Unsupported type " + type); } var columnMeta = new ColumnMeta(); - columnMeta.name = propertyType.getName(); + columnMeta.name = column.name(); columnMeta.type = type.byteValue(); return columnMeta; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDatabase.java index b405315a..1b05afe5 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageDatabase.java @@ -13,17 +13,16 @@ package org.apache.baremaps.storage.geopackage; - -import java.io.IOException; import java.nio.file.Path; -import java.util.stream.Stream; +import java.util.Collection; import mil.nga.geopackage.GeoPackage; import mil.nga.geopackage.GeoPackageManager; -import org.apache.baremaps.feature.ReadableAggregate; -import org.apache.baremaps.feature.Resource; +import org.apache.baremaps.dataframe.DataFrame; +import org.apache.baremaps.dataframe.DataFrameException; +import org.apache.baremaps.dataframe.DataStore; -public class GeoPackageDatabase implements ReadableAggregate, AutoCloseable { +public class GeoPackageDatabase implements DataStore, AutoCloseable { private final GeoPackage geoPackage; @@ -32,13 +31,33 @@ public class GeoPackageDatabase implements ReadableAggregate, AutoCloseable { } @Override - public Stream<Resource> read() throws IOException { + public void close() throws Exception { + geoPackage.close(); + } + + @Override + public Collection<DataFrame> list() throws DataFrameException { return geoPackage.getFeatureTables().stream() - .map(table -> new GeoPackageTable(geoPackage.getFeatureDao(table))); + .map(table -> new GeoPackageTable(geoPackage.getFeatureDao(table))) + .map(DataFrame.class::cast) + .toList(); } @Override - public void close() throws Exception { - geoPackage.close(); + public DataFrame get(String name) throws DataFrameException { + return list().stream() + .filter(dataFrame -> dataFrame.dataType().name().equals(name)) + .findFirst() + .orElseThrow(() -> new DataFrameException()); + } + + @Override + public void add(DataFrame value) throws DataFrameException { + throw new UnsupportedOperationException(); + } + + @Override + public void remove(String name) throws DataFrameException { + throw new UnsupportedOperationException(); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java index 9fb0e186..b406f29c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geopackage/GeoPackageTable.java @@ -13,47 +13,33 @@ package org.apache.baremaps.storage.geopackage; - -import java.io.IOException; import java.util.*; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; import mil.nga.geopackage.features.user.FeatureColumn; import mil.nga.geopackage.features.user.FeatureDao; import mil.nga.geopackage.features.user.FeatureResultSet; import mil.nga.geopackage.geom.GeoPackageGeometryData; -import org.apache.baremaps.feature.*; -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.GeometryCollection; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.LineString; -import org.locationtech.jts.geom.LinearRing; -import org.locationtech.jts.geom.MultiLineString; -import org.locationtech.jts.geom.MultiPoint; -import org.locationtech.jts.geom.MultiPolygon; -import org.locationtech.jts.geom.Point; -import org.locationtech.jts.geom.Polygon; -import org.locationtech.jts.geom.PrecisionModel; - -public class GeoPackageTable implements ReadableFeatureSet { +import org.apache.baremaps.collection.AbstractDataCollection; +import org.apache.baremaps.dataframe.*; +import org.locationtech.jts.geom.*; + +public class GeoPackageTable extends AbstractDataCollection<Row> implements DataFrame { private final FeatureDao featureDao; - private final FeatureType featureType; + private final DataType featureType; private final GeometryFactory geometryFactory; protected GeoPackageTable(FeatureDao featureDao) { this.featureDao = featureDao; var name = featureDao.getTableName(); - var properties = new HashMap<String, PropertyType>(); + var columns = new ArrayList<Column>(); for (FeatureColumn column : featureDao.getColumns()) { var propertyName = column.getName(); var propertyType = classType(column); - properties.put(propertyName, new PropertyType(propertyName, propertyType)); + columns.add(new ColumnImpl(propertyName, propertyType)); } - featureType = new FeatureTypeImpl(name, properties); + featureType = new DataTypeImpl(name, columns); geometryFactory = new GeometryFactory(new PrecisionModel(), (int) featureDao.getSrs().getId()); } @@ -66,27 +52,31 @@ public class GeoPackageTable implements ReadableFeatureSet { } @Override - public FeatureType getType() throws IOException { - return featureType; + public Iterator<Row> iterator() { + return new FeatureIterator(featureDao.queryForAll(), featureType); + } + + @Override + public long sizeAsLong() { + return 0; } @Override - public Stream<Feature> read() throws IOException { - var featureIterator = new FeatureIterator(featureDao.queryForAll(), featureType); - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(featureIterator, 0), false); + public DataType dataType() { + return null; } - public class FeatureIterator implements Iterator<Feature> { + public class FeatureIterator implements Iterator<Row> { private final FeatureResultSet featureResultSet; - private final FeatureType featureType; + private final DataType dataType; private boolean hasNext; - public FeatureIterator(FeatureResultSet featureResultSet, FeatureType featureType) { + public FeatureIterator(FeatureResultSet featureResultSet, DataType dataType) { this.featureResultSet = featureResultSet; - this.featureType = featureType; + this.dataType = dataType; this.hasNext = featureResultSet.moveToFirst(); } @@ -96,19 +86,19 @@ public class GeoPackageTable implements ReadableFeatureSet { } @Override - public Feature next() { + public Row next() { if (!hasNext) { throw new NoSuchElementException(); } - Feature feature = featureType.newInstance(); + Row row = dataType.newInstance(); for (FeatureColumn featureColumn : featureResultSet.getColumns().getColumns()) { var value = featureResultSet.getValue(featureColumn); if (value != null) { - feature.setProperty(featureColumn.getName(), asValue(value)); + row.set(featureColumn.getName(), asValue(value)); } } hasNext = featureResultSet.moveToNext(); - return feature; + return row; } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataFrame.java similarity index 51% rename from baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTable.java rename to baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataFrame.java index d7f8407a..3c49de7c 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTable.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDataFrame.java @@ -14,27 +14,32 @@ package org.apache.baremaps.storage.postgres; -import java.io.IOException; -import java.util.stream.Stream; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.WritableFeatureSet; +import java.util.Iterator; +import org.apache.baremaps.collection.AbstractDataCollection; +import org.apache.baremaps.dataframe.DataFrame; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; -public class PostgresTable implements WritableFeatureSet { +public class PostgresDataFrame extends AbstractDataCollection<Row> implements DataFrame { - private final FeatureType featureType; + private final DataType dataType; - public PostgresTable(FeatureType featureType) { - this.featureType = featureType; + public PostgresDataFrame(DataType dataType) { + this.dataType = dataType; } @Override - public FeatureType getType() throws IOException { - return featureType; + public Iterator<Row> iterator() { + return null; } @Override - public void write(Stream<Feature> features) throws IOException { - throw new UnsupportedOperationException(); + public long sizeAsLong() { + return 0; + } + + @Override + public DataType dataType() { + return dataType(); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java index f7214c27..76df463a 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresDatabase.java @@ -13,13 +13,12 @@ package org.apache.baremaps.storage.postgres; - import de.bytefish.pgbulkinsert.pgsql.handlers.*; -import java.io.IOException; import java.sql.SQLException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,12 +26,12 @@ import java.util.stream.Collectors; import javax.sql.DataSource; import org.apache.baremaps.database.copy.CopyWriter; import org.apache.baremaps.database.copy.PostgisGeometryValueHandler; -import org.apache.baremaps.feature.*; +import org.apache.baremaps.dataframe.*; import org.locationtech.jts.geom.*; import org.postgresql.PGConnection; import org.postgresql.copy.PGCopyOutputStream; -public class PostgresDatabase implements WritableAggregate { +public class PostgresDatabase implements DataStore { // TODO: Instantiate with Map.of() private static Map<Class, String> typeToName = new HashMap<>(); @@ -90,108 +89,112 @@ public class PostgresDatabase implements WritableAggregate { this.dataSource = dataSource; } - private FeatureType createFeatureType(FeatureType featureType) { - var name = featureType.getName().replaceAll("[^a-zA-Z0-9]", "_"); - var properties = featureType.getPropertyTypes().values().stream() - .filter(type -> typeToName.containsKey(type.getType())) - .collect(Collectors.toMap(k -> k.getName(), v -> v)); - return new FeatureTypeImpl(name, properties); + private DataType adaptDataType(DataType datatype) { + var name = datatype.name().replaceAll("[^a-zA-Z0-9]", "_"); + var properties = datatype.columns().stream() + .filter(columnType -> typeToName.containsKey(columnType.type())) + .toList(); + return new DataTypeImpl(name, properties); } @Override - public void write(Resource resource) throws IOException { - if (resource instanceof ReadableFeatureSet featureSetReader) { - try (var connection = dataSource.getConnection()) { - var featureType = createFeatureType(featureSetReader.getType()); - - // Drop the table if it exists - var dropQuery = dropTable(featureType); - try (var dropStatement = connection.prepareStatement(dropQuery)) { - dropStatement.execute(); - } + public Collection<DataFrame> list() throws DataFrameException { + throw new UnsupportedOperationException(); + } - // Create the table - var createQuery = createTable(featureType); - try (var createStatement = connection.prepareStatement(createQuery)) { - createStatement.execute(); - } + @Override + public DataFrame get(String name) throws DataFrameException { + throw new UnsupportedOperationException(); + } - // Populate the table with a copy query - PGConnection pgConnection = connection.unwrap(PGConnection.class); - var copyQuery = copyTable(featureType); - try (var writer = new CopyWriter(new PGCopyOutputStream(pgConnection, copyQuery))) { - writer.writeHeader(); - var featureIterator = featureSetReader.read().iterator(); - while (featureIterator.hasNext()) { - var feature = featureIterator.next(); - var attributes = getAttributes(featureType); - writer.startRow(attributes.size()); - for (var attribute : attributes) { - var name = attribute.getName().toString(); - var value = feature.getProperty(name); - if (value == null) { - writer.writeNull(); - } else { - writer.write(typeToHandler.get(value.getClass()), value); - } + @Override + public void add(DataFrame dataFrame) { + try (var connection = dataSource.getConnection()) { + var dataType = adaptDataType(dataFrame.dataType()); + + // Drop the table if it exists + var dropQuery = dropTable(dataType); + try (var dropStatement = connection.prepareStatement(dropQuery)) { + dropStatement.execute(); + } + + // Create the table + var createQuery = createTable(dataType); + try (var createStatement = connection.prepareStatement(createQuery)) { + createStatement.execute(); + } + + // Populate the table with a copy query + PGConnection pgConnection = connection.unwrap(PGConnection.class); + var copyQuery = copyTable(dataType); + try (var writer = new CopyWriter(new PGCopyOutputStream(pgConnection, copyQuery))) { + writer.writeHeader(); + var featureIterator = dataFrame.iterator(); + while (featureIterator.hasNext()) { + var feature = featureIterator.next(); + var attributes = getAttributes(dataType); + writer.startRow(attributes.size()); + for (var attribute : attributes) { + var name = attribute.name().toString(); + var value = feature.get(name); + if (value == null) { + writer.writeNull(); + } else { + writer.write(typeToHandler.get(value.getClass()), value); } } } - } catch (SQLException e) { - throw new IOException(e); } + } catch (Exception e) { + throw new RuntimeException(e); } } - private List<PropertyType> getAttributes(FeatureType featureType) { - return featureType.getPropertyTypes().values().stream() + private List<Column> getAttributes(DataType dataType) { + return dataType.columns().stream() .filter(this::isSupported).collect(Collectors.toList()); } - private boolean isSupported(PropertyType propertyType) { - return typeToName.containsKey(propertyType.getType()); + private boolean isSupported(Column column) { + return typeToName.containsKey(column.type()); } - private String createTable(FeatureType featureType) { + private String createTable(DataType dataType) { StringBuilder builder = new StringBuilder(); builder.append("CREATE TABLE "); - builder.append(featureType.getName()); + builder.append(dataType.name()); builder.append(" ("); - builder.append(featureType.getPropertyTypes().values().stream() - .map(attributeType -> attributeType.getName() - + " " + typeToName.get(attributeType.getType())) + builder.append(dataType.columns().stream() + .map(column -> column.name() + + " " + typeToName.get(column.type())) .collect(Collectors.joining(", "))); builder.append(")"); return builder.toString(); } - private String copyTable(FeatureType featureType) { + private String copyTable(DataType dataType) { StringBuilder builder = new StringBuilder(); builder.append("COPY "); - builder.append(featureType.getName()); + builder.append(dataType.name()); builder.append(" ("); - builder.append(featureType.getPropertyTypes().values().stream() - .map(propertyType -> propertyType.getName()) + builder.append(dataType.columns().stream() + .map(column -> column.name()) .collect(Collectors.joining(", "))); builder.append(") FROM STDIN BINARY"); return builder.toString(); } @Override - public void remove(Resource resource) throws IOException { - if (resource instanceof FeatureSet featureSet) { - var type = featureSet.getType(); - try (var connection = dataSource.getConnection(); - var statement = connection.createStatement()) { - statement.executeQuery(String.format("DROP TABLE IF EXISTS %s CASCADE", type.getName())); - } catch (SQLException e) { - throw new RuntimeException(e); - } + public void remove(String name) { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeQuery(String.format("DROP TABLE IF EXISTS %s CASCADE", name)); + } catch (SQLException e) { + throw new RuntimeException(e); } } - private String dropTable(FeatureType type) { - return String.format("DROP TABLE IF EXISTS %s CASCADE", type.getName()); + private String dropTable(DataType type) { + return String.format("DROP TABLE IF EXISTS %s CASCADE", type.name()); } - } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java new file mode 100644 index 00000000..8226ea26 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataFrame.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.storage.shapefile; + + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.NoSuchElementException; +import org.apache.baremaps.collection.AbstractDataCollection; +import org.apache.baremaps.dataframe.DataFrame; +import org.apache.baremaps.dataframe.DataFrameException; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; +import org.apache.baremaps.storage.shapefile.internal.InputFeatureStream; +import org.apache.baremaps.storage.shapefile.internal.ShapefileReader; + +public class ShapefileDataFrame extends AbstractDataCollection<Row> + implements DataFrame, AutoCloseable { + + private final ShapefileReader shapeFile; + + public ShapefileDataFrame(Path file) { + this.shapeFile = new ShapefileReader(file.toString()); + } + + @Override + public DataType dataType() throws DataFrameException { + try (var input = shapeFile.read()) { + return input.getDataType(); + } catch (IOException e) { + throw new DataFrameException(e); + } + } + + @Override + public Iterator<Row> iterator() { + try { + return new RowIterator(shapeFile.read()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long sizeAsLong() { + return 0; + } + + @Override + public void close() throws Exception { + + } + + static class RowIterator implements Iterator<Row> { + + private final InputFeatureStream inputFeatureStream; + + private Row next; + + public RowIterator(InputFeatureStream inputFeatureStream) { + this.inputFeatureStream = inputFeatureStream; + } + + + @Override + public boolean hasNext() { + try { + if (next == null) { + next = inputFeatureStream.readFeature(); + } + return true; + } catch (IOException exception) { + return false; + } + } + + @Override + public Row next() { + try { + if (next == null) { + next = inputFeatureStream.readFeature(); + } + Row current = next; + next = null; + return current; + } catch (Exception e) { + throw new NoSuchElementException(); + } + } + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDirectory.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDirectory.java index a187b7e1..919bc062 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDirectory.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDirectory.java @@ -17,11 +17,12 @@ package org.apache.baremaps.storage.shapefile; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.stream.Stream; -import org.apache.baremaps.feature.ReadableAggregate; -import org.apache.baremaps.feature.Resource; +import java.util.Collection; +import org.apache.baremaps.dataframe.DataFrame; +import org.apache.baremaps.dataframe.DataFrameException; +import org.apache.baremaps.dataframe.DataStore; -public class ShapefileDirectory implements ReadableAggregate { +public class ShapefileDirectory implements DataStore { private final Path directory; @@ -30,10 +31,33 @@ public class ShapefileDirectory implements ReadableAggregate { } @Override - public Stream<Resource> read() throws IOException { - return Files.list(directory) - .filter(file -> file.toString().toLowerCase().endsWith(".shp")) - .map(file -> new ShapefileFeatureSet(file)); + public Collection<DataFrame> list() { + try { + return Files.list(directory) + .filter(file -> file.toString().toLowerCase().endsWith(".shp")) + .map(file -> new ShapefileDataFrame(file)) + .map(DataFrame.class::cast) + .toList(); + } catch (IOException e) { + throw new DataFrameException(e); + } + } + + @Override + public DataFrame get(String name) { + return list().stream() + .filter(dataFrame -> dataFrame.dataType().name().equals(name)) + .findFirst() + .orElseThrow(() -> new DataFrameException()); + } + @Override + public void add(DataFrame value) { + throw new UnsupportedOperationException(); + } + + @Override + public void remove(String name) { + throw new UnsupportedOperationException(); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileFeatureSet.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileFeatureSet.java deleted file mode 100644 index bf688b8b..00000000 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileFeatureSet.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.baremaps.storage.shapefile; - - - -import java.io.IOException; -import java.nio.file.Path; -import java.util.Spliterator; -import java.util.function.Consumer; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.ReadableFeatureSet; -import org.apache.baremaps.storage.shapefile.internal.InputFeatureStream; -import org.apache.baremaps.storage.shapefile.internal.ShapefileReader; - -public class ShapefileFeatureSet implements ReadableFeatureSet, AutoCloseable { - - private final ShapefileReader shapeFile; - - public ShapefileFeatureSet(Path file) { - this.shapeFile = new ShapefileReader(file.toString()); - } - - @Override - public FeatureType getType() throws IOException { - try (var input = shapeFile.read()) { - return input.getFeaturesType(); - } - } - - @Override - public Stream<Feature> read() throws IOException { - return StreamSupport.stream(new FeatureSpliterator(shapeFile.read()), false); - } - - static class FeatureSpliterator implements Spliterator<Feature> { - - private final InputFeatureStream inputFeatureStream; - - public FeatureSpliterator(InputFeatureStream inputFeatureStream) { - this.inputFeatureStream = inputFeatureStream; - } - - @Override - public boolean tryAdvance(Consumer<? super Feature> action) { - try { - var feature = inputFeatureStream.readFeature(); - if (feature != null) { - action.accept(feature); - return true; - } else { - return false; - } - } catch (Exception e) { - return false; - } - } - - @Override - public Spliterator<Feature> trySplit() { - return null; - } - - @Override - public long estimateSize() { - return Long.MAX_VALUE; - } - - @Override - public int characteristics() { - return 0; - } - } - - @Override - public void close() throws Exception { - - } -} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java index 13eea12e..e48a1e0d 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java @@ -22,7 +22,7 @@ import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.text.MessageFormat; import java.util.*; -import org.apache.baremaps.feature.Feature; +import org.apache.baremaps.dataframe.Row; /** * Reader of a Database Binary content. @@ -99,9 +99,9 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { /** * Load a row into a feature. * - * @param feature Feature to fill. + * @param row Feature to fill. */ - public void loadRowIntoFeature(Feature feature) { + public void loadRow(Row row) { // TODO: ignore deleted records getByteBuffer().get(); // denotes whether deleted or current // read first part of record @@ -138,7 +138,7 @@ public class DbaseByteReader extends CommonByteReader implements AutoCloseable { case DateTime -> value; }; - feature.setProperty(fd.getName(), object); + row.set(fd.getName(), object); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java index 89849c6a..7fb8d72e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/InputFeatureStream.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.io.InputStream; import java.sql.SQLFeatureNotSupportedException; import java.util.List; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; /** * Input Stream of features. @@ -48,7 +48,7 @@ public class InputFeatureStream extends InputStream { private boolean hasShapefileIndex; /** Type of the features contained in this shapefile. */ - private FeatureType featuresType; + private DataType dataType; /** Shapefile reader. */ private ShapefileByteReader shapefileReader; @@ -74,7 +74,7 @@ public class InputFeatureStream extends InputStream { this.shapefileReader = new ShapefileByteReader(this.shapefile, this.databaseFile, this.shapefileIndex); - this.featuresType = this.shapefileReader.getFeaturesType(); + this.dataType = this.shapefileReader.getDataType(); } /** @@ -113,7 +113,7 @@ public class InputFeatureStream extends InputStream { * @throws ShapefileException if the current connection used to query the shapefile has been * closed. */ - public Feature readFeature() throws ShapefileException { + public Row readFeature() throws ShapefileException { return internalReadFeature(); } @@ -122,8 +122,8 @@ public class InputFeatureStream extends InputStream { * * @return Features type. */ - public FeatureType getFeaturesType() { - return featuresType; + public DataType getDataType() { + return dataType; } /** @@ -160,14 +160,14 @@ public class InputFeatureStream extends InputStream { * @throws SQLFeatureNotSupportedException if a SQL ability is not currently available through * this driver. */ - private Feature internalReadFeature() throws ShapefileException { + private Row internalReadFeature() throws ShapefileException { if (!this.dbaseReader.nextRowAvailable()) { return null; } - Feature feature = this.featuresType.newInstance(); - this.dbaseReader.loadRowIntoFeature(feature); + Row row = this.dataType.newInstance(); + this.dbaseReader.loadRow(row); this.shapefileReader.setRowNum(this.dbaseReader.getRowNum()); - this.shapefileReader.completeFeature(feature); - return feature; + this.shapefileReader.completeFeature(row); + return row; } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java index 8dbb3fbd..a7572903 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java @@ -19,10 +19,7 @@ import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.*; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.FeatureTypeImpl; -import org.apache.baremaps.feature.PropertyType; +import org.apache.baremaps.dataframe.*; import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateList; @@ -48,7 +45,7 @@ public class ShapefileByteReader extends CommonByteReader { private List<DBaseFieldDescriptor> databaseFieldsDescriptors; /** Type of the features contained in this shapefile. */ - private FeatureType featuresType; + private DataType dataType; /** Shapefile index. */ private File shapeFileIndex; @@ -83,7 +80,7 @@ public class ShapefileByteReader extends CommonByteReader { loadShapefileIndexes(); } - this.featuresType = getFeatureType(shapefile.getName()); + this.dataType = getDataType(shapefile.getName()); } /** @@ -109,8 +106,8 @@ public class ShapefileByteReader extends CommonByteReader { * * @return Features type. */ - public FeatureType getFeaturesType() { - return this.featuresType; + public DataType getDataType() { + return this.dataType; } /** @@ -119,13 +116,12 @@ public class ShapefileByteReader extends CommonByteReader { * @param name Name of the field. * @return The feature type. */ - private FeatureType getFeatureType(final String name) { + private DataType getDataType(final String name) { Objects.requireNonNull(name, "The feature name cannot be null."); - var properties = new HashMap<String, PropertyType>(); + var columns = new ArrayList<Column>(); for (int i = 0; i < databaseFieldsDescriptors.size(); i++) { var fieldDescriptor = this.databaseFieldsDescriptors.get(i); - var propertyName = fieldDescriptor.getName(); var propertyType = switch (fieldDescriptor.getType()) { case Character -> String.class; @@ -147,13 +143,13 @@ public class ShapefileByteReader extends CommonByteReader { case DateTime -> String.class; }; - properties.put(propertyName, new PropertyType(propertyName, propertyType)); + columns.add(new ColumnImpl(propertyName, propertyType)); } - // Add geometry field. - properties.put(GEOMETRY_NAME, new PropertyType(GEOMETRY_NAME, Geometry.class)); + // Add geometry column. + columns.add(new ColumnImpl(GEOMETRY_NAME, Geometry.class)); - return new FeatureTypeImpl(name, properties); + return new DataTypeImpl(name, columns); } /** Load shapefile descriptor. */ @@ -260,7 +256,7 @@ public class ShapefileByteReader extends CommonByteReader { * * @param feature Feature to complete. */ - public void completeFeature(Feature feature) throws ShapefileException { + public void completeFeature(Row feature) throws ShapefileException { // insert points into some type of list int RecordNumber = getByteBuffer().getInt(); int ContentLength = getByteBuffer().getInt(); @@ -300,11 +296,11 @@ public class ShapefileByteReader extends CommonByteReader { * * @param feature Feature to fill. */ - private void loadPointFeature(Feature feature) { + private void loadPointFeature(Row feature) { double x = getByteBuffer().getDouble(); double y = getByteBuffer().getDouble(); Point pnt = geometryFactory.createPoint(new Coordinate(x, y)); - feature.setProperty(GEOMETRY_NAME, pnt); + feature.set(GEOMETRY_NAME, pnt); } /** @@ -312,7 +308,7 @@ public class ShapefileByteReader extends CommonByteReader { * * @param feature Feature to fill. */ - private void loadPolygonFeature(Feature feature) { + private void loadPolygonFeature(Row feature) { double xmin = getByteBuffer().getDouble(); double ymin = getByteBuffer().getDouble(); double xmax = getByteBuffer().getDouble(); @@ -323,7 +319,7 @@ public class ShapefileByteReader extends CommonByteReader { Geometry multiPolygon = readMultiplePolygon(numParts, numPoints); - feature.setProperty(GEOMETRY_NAME, multiPolygon); + feature.set(GEOMETRY_NAME, multiPolygon); } /** @@ -388,7 +384,7 @@ public class ShapefileByteReader extends CommonByteReader { * * @param feature Feature to fill. */ - private void loadPolylineFeature(Feature feature) { + private void loadPolylineFeature(Row feature) { /* double xmin = */ getByteBuffer().getDouble(); /* double ymin = */ getByteBuffer().getDouble(); /* double xmax = */ getByteBuffer().getDouble(); @@ -420,7 +416,7 @@ public class ShapefileByteReader extends CommonByteReader { } } - feature.setProperty(GEOMETRY_NAME, + feature.set(GEOMETRY_NAME, geometryFactory.createLineString(coordinates.toCoordinateArray())); } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java index 97625402..9d5314ec 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileReader.java @@ -18,7 +18,7 @@ import java.io.File; import java.io.IOException; import java.util.List; import java.util.Objects; -import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.dataframe.DataType; /** * Provides a ShapeFile Reader. @@ -44,7 +44,7 @@ public class ShapefileReader { private File shapeFileIndex; /** Type of the features contained in this shapefile. */ - private FeatureType featuresType; + private DataType dataType; /** Shapefile descriptor. */ private ShapefileDescriptor shapefileDescriptor; @@ -117,8 +117,8 @@ public class ShapefileReader { * * @return Feature type. */ - public FeatureType getFeaturesType() { - return this.featuresType; + public DataType getDataType() { + return this.dataType; } /** @@ -174,7 +174,7 @@ public class ShapefileReader { public InputFeatureStream read() throws IOException { InputFeatureStream is = new InputFeatureStream(this.shapefile, this.databaseFile, this.shapeFileIndex); - this.featuresType = is.getFeaturesType(); + this.dataType = is.getDataType(); this.shapefileDescriptor = is.getShapefileDescriptor(); this.databaseFieldsDescriptors = is.getDatabaseFieldsDescriptors(); return is; diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java index 17a8d336..b173a0c2 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java @@ -12,15 +12,13 @@ package org.apache.baremaps.workflow.tasks; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; -import org.apache.baremaps.feature.FeatureTypeImpl; -import org.apache.baremaps.feature.PropertyType; +import org.apache.baremaps.dataframe.*; import org.locationtech.jts.geom.Geometry; -public class Entity implements Feature { +public class Entity implements Row { private long id; @@ -36,23 +34,23 @@ public class Entity implements Feature { @Override - public FeatureType getType() { - var map = new HashMap<String, PropertyType>(); - map.put("id", new PropertyType<>("id", Long.class)); + public DataType dataType() { + var columns = new ArrayList<Column>(); + columns.add(new ColumnImpl("id", Long.class)); + columns.add(new ColumnImpl("geometry", Geometry.class)); for (var entry : tags.entrySet()) { - map.put(entry.getKey(), new PropertyType<>(entry.getKey(), String.class)); + columns.add(new ColumnImpl(entry.getKey(), String.class)); } - map.put("geometry", new PropertyType<>("geometry", Geometry.class)); - return new FeatureTypeImpl("entity", map); + return new DataTypeImpl("entity", columns); } @Override - public void setProperty(String name, Object value) { + public void set(String name, Object value) { tags.put(name, value.toString()); } @Override - public Object getProperty(String name) { + public Object get(String name) { if (name.equals("id")) { return id; } else if (name.equals("geometry")) { @@ -63,11 +61,11 @@ public class Entity implements Feature { } @Override - public Map<String, Object> getProperties() { - var map = new HashMap<String, Object>(); - map.put("id", id); - map.putAll(tags); - map.put("geometry", geometry); + public List<Object> values() { + var map = new ArrayList(); + map.add(id); + map.add(geometry); + map.addAll(tags.values()); return map; } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java index 6ec56f67..735127df 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java @@ -13,9 +13,8 @@ package org.apache.baremaps.workflow.tasks; import java.nio.file.Path; -import org.apache.baremaps.feature.ReadableFeatureSet; import org.apache.baremaps.openstreetmap.utils.ProjectionTransformer; -import org.apache.baremaps.storage.FeatureSetProjectionTransform; +import org.apache.baremaps.storage.DataFrameProjectionTransform; import org.apache.baremaps.storage.geopackage.GeoPackageDatabase; import org.apache.baremaps.storage.postgres.PostgresDatabase; import org.apache.baremaps.workflow.Task; @@ -37,12 +36,10 @@ public record ImportGeoPackage(Path file, String database, Integer sourceSRID, I try (var geoPackageStore = new GeoPackageDatabase(path)) { var dataSource = context.getDataSource(database); var postgresDatabase = new PostgresDatabase(dataSource); - for (var resource : geoPackageStore.read().toList()) { - if (resource instanceof ReadableFeatureSet featureSet) { - var transformer = new ProjectionTransformer(sourceSRID, targetSRID); - var transformedFeatureSet = new FeatureSetProjectionTransform(featureSet, transformer); - postgresDatabase.write(transformedFeatureSet); - } + for (var dataFrame : geoPackageStore.list()) { + var transformer = new ProjectionTransformer(sourceSRID, targetSRID); + var transformedDataFrame = new DataFrameProjectionTransform(dataFrame, transformer); + postgresDatabase.add(transformedDataFrame); } logger.info("Finished importing {} into {}", file, database); } catch (Exception e) { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java index ca0b3208..4582efe6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java @@ -14,9 +14,9 @@ package org.apache.baremaps.workflow.tasks; import java.nio.file.Path; import org.apache.baremaps.openstreetmap.utils.ProjectionTransformer; -import org.apache.baremaps.storage.FeatureSetProjectionTransform; +import org.apache.baremaps.storage.DataFrameProjectionTransform; import org.apache.baremaps.storage.postgres.PostgresDatabase; -import org.apache.baremaps.storage.shapefile.ShapefileFeatureSet; +import org.apache.baremaps.storage.shapefile.ShapefileDataFrame; import org.apache.baremaps.workflow.Task; import org.apache.baremaps.workflow.WorkflowContext; import org.apache.baremaps.workflow.WorkflowException; @@ -33,10 +33,10 @@ public record ImportShapefile(Path file, String database, Integer sourceSRID, In public void execute(WorkflowContext context) throws Exception { logger.info("Importing {} into {}", file, database); var path = file.toAbsolutePath(); - try (var featureSet = new ShapefileFeatureSet(path)) { + try (var featureSet = new ShapefileDataFrame(path)) { var dataSource = context.getDataSource(database); var postgresDatabase = new PostgresDatabase(dataSource); - postgresDatabase.write(new FeatureSetProjectionTransform( + postgresDatabase.add(new DataFrameProjectionTransform( featureSet, new ProjectionTransformer(sourceSRID, targetSRID))); logger.info("Finished importing {} into {}", file, database); } catch (Exception e) { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java index b39a06f9..f2cc47b3 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java @@ -15,15 +15,15 @@ package org.apache.baremaps.workflow.tasks; import java.nio.file.Path; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.baremaps.collection.AppendOnlyBuffer; import org.apache.baremaps.collection.algorithm.UnionStream; import org.apache.baremaps.collection.memory.MemoryMappedFile; -import org.apache.baremaps.feature.*; +import org.apache.baremaps.dataframe.*; import org.apache.baremaps.mvt.expression.Expressions.Expression; import org.apache.baremaps.storage.postgres.PostgresDatabase; import org.apache.baremaps.workflow.Task; @@ -53,25 +53,42 @@ public record TransformEntityCollection(Path collection, String database, public void execute(WorkflowContext context) throws Exception { logger.info("Transform {} with {}", collection, recipe); - var featureType = new FeatureTypeImpl(recipe.name, propertyTypes()); + var dataType = new DataTypeImpl(recipe.name, columns()); var groups = new AppendOnlyBuffer<>(new EntityDataType(), new MemoryMappedFile(collection)) .stream() .filter(this::filter) .collect(Collectors.groupingBy(this::propertyValues)); - var featureStream = groups.entrySet().stream().flatMap(entry -> { - var group = entry.getKey(); - var entities = entry.getValue(); - var geometries = simplify(entities.stream().map(Entity::getGeometry)); - return geometries.map(geometry -> new Entity(0, group, geometry)); + var entityStream = groups.entrySet().stream().flatMap(entry -> { + var tags = IntStream.range(0, recipe.groupBy.size()).boxed() + .collect(Collectors.toMap(i -> recipe.groupBy.get(i), i -> entry.getKey().get(i))); + var geometries = entry.getValue(); + var simplified = simplify(geometries.stream().map(Entity::getGeometry)); + return simplified.map(geometry -> (Row) new Entity(0, tags, geometry)); }); - var featureSet = new ReadableFeatureStream(featureType, featureStream); + var dataFrame = new AbstractDataFrame() { + + @Override + public DataType dataType() { + return dataType; + } + + @Override + public long sizeAsLong() { + return 0; + } + + @Override + public Iterator<Row> iterator() { + return entityStream.iterator(); + } + }; var dataSource = context.getDataSource(database); var postgresDatabase = new PostgresDatabase(dataSource); - postgresDatabase.write(featureSet); + postgresDatabase.add(dataFrame); } private Stream<Geometry> simplify(Stream<Geometry> geometries) { @@ -86,19 +103,19 @@ public record TransformEntityCollection(Path collection, String database, return recipe.groupBy() == null ? List.of() : recipe.groupBy(); } - private Map<String, PropertyType> propertyTypes() { - var map = new HashMap<String, PropertyType>(); + private List<Column> columns() { + var list = new ArrayList<Column>(); for (var property : groupBy()) { - map.put(property, new PropertyType<>(property, String.class)); + list.add(new ColumnImpl(property, String.class)); } - map.put("geometry", new PropertyType<>("geometry", Geometry.class)); - return map; + list.add(new ColumnImpl("geometry", Geometry.class)); + return list; } - private Map<String, String> propertyValues(Entity entity) { - var map = new HashMap<String, String>(); + private List<String> propertyValues(Entity entity) { + var map = new ArrayList<String>(); for (var property : groupBy()) { - map.put(property, entity.getProperty(property).toString()); + map.add(entity.get(property).toString()); } return map; } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/mvt/ExpressionsTest.java b/baremaps-core/src/test/java/org/apache/baremaps/mvt/ExpressionsTest.java index f6bd831b..83e7b80b 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/mvt/ExpressionsTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/mvt/ExpressionsTest.java @@ -17,34 +17,34 @@ import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; import java.util.List; import java.util.Map; -import org.apache.baremaps.feature.Feature; -import org.apache.baremaps.feature.FeatureType; +import org.apache.baremaps.dataframe.DataType; +import org.apache.baremaps.dataframe.Row; import org.apache.baremaps.mvt.expression.Expressions; import org.apache.baremaps.mvt.expression.Expressions.*; import org.junit.jupiter.api.Test; class ExpressionsTest { - record FeatureMock(Map<String, Object> properties) implements Feature { + record RowMock(Map<String, Object> properties) implements Row { @Override - public FeatureType getType() { + public DataType dataType() { throw new UnsupportedOperationException(); } @Override - public void setProperty(String name, Object value) { - properties.put(name, value); + public List<Object> values() { + return properties.values().stream().toList(); } @Override - public Object getProperty(String name) { - return properties.get(name); + public void set(String name, Object value) { + properties.put(name, value); } @Override - public Map<String, Object> getProperties() { - return properties; + public Object get(String name) { + return properties.get(name); } } @@ -68,15 +68,15 @@ class ExpressionsTest { @Test public void get() throws IOException { assertEquals("value", - new Get("key").evaluate(new FeatureMock(Map.of("key", "value")))); - assertEquals(null, new Get("key").evaluate(new FeatureMock(Map.of()))); + new Get("key").evaluate(new RowMock(Map.of("key", "value")))); + assertEquals(null, new Get("key").evaluate(new RowMock(Map.of()))); } @Test public void has() throws IOException { assertEquals(true, - new Has("key").evaluate(new FeatureMock(Map.of("key", "value")))); - assertEquals(false, new Has("key").evaluate(new FeatureMock(Map.of()))); + new Has("key").evaluate(new RowMock(Map.of("key", "value")))); + assertEquals(false, new Has("key").evaluate(new RowMock(Map.of()))); } @Test diff --git a/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java b/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java index 3c17ea83..a47088be 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufFeatureSetTest.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.StandardOpenOption; - import org.apache.baremaps.testing.TestFiles; import org.junit.jupiter.api.Test; @@ -29,8 +28,8 @@ class FlatGeoBufFeatureSetTest { try (var channel = FileChannel.open(TestFiles.resolve("countries.fgb"))) { var featureSet = new FlatGeoBufFeatureSet(channel); var featureType = featureSet.getType(); - assertEquals(featureType.getName(), null); - assertEquals(featureType.getPropertyTypes().size(), 2); + assertEquals(featureType.name(), null); + assertEquals(featureType.columns().size(), 2); } } @@ -48,7 +47,7 @@ class FlatGeoBufFeatureSetTest { var file = Files.createTempFile("countries", ".fgb"); file.toFile().deleteOnExit(); try (var channel1 = FileChannel.open(TestFiles.resolve("countries.fgb")); - var channel2 = FileChannel.open(file, StandardOpenOption.WRITE)) { + var channel2 = FileChannel.open(file, StandardOpenOption.WRITE)) { var featureSet1 = new FlatGeoBufFeatureSet(channel1); var featureList = featureSet1.read().stream().toList(); var featureSet2 = new FlatGeoBufFeatureSet(channel2, featureSet1.getType());
