This is an automated email from the ASF dual-hosted git repository. desruisseaux pushed a commit to branch geoapi-4.0 in repository https://gitbox.apache.org/repos/asf/sis.git
commit ab1c1baf1fd746ded954c9111a8dcf1ba6887a5f Author: Martin Desruisseaux <martin.desruisse...@geomatys.com> AuthorDate: Fri Aug 19 19:35:57 2022 +0200 First draft of `CoverageAggregator` for creating a concatenated grid coverage. The concatenation happens in a single dimension, for example the time, in which case the concatenation of many grid coverages create a time series. There is no JUnit test yet (will need to be added later). --- .../sis/internal/storage/MemoryGridResource.java | 4 +- .../aggregate/ConcatenatedGridCoverage.java | 128 ++++++++++ .../aggregate/ConcatenatedGridResource.java | 275 +++++++++++++++++++++ .../storage/aggregate/CoverageAggregator.java | 126 ++++++++++ .../storage/aggregate/DimensionSelector.java | 150 +++++++++++ .../sis/internal/storage/aggregate/GridSlice.java | 231 +++++++++++++++++ .../storage/aggregate/GridSliceLocator.java | 193 +++++++++++++++ .../sis/internal/storage/aggregate/Group.java | 92 +++++++ .../internal/storage/aggregate/GroupAggregate.java | 238 ++++++++++++++++++ .../sis/internal/storage/aggregate/GroupByCRS.java | 103 ++++++++ .../internal/storage/aggregate/GroupBySample.java | 99 ++++++++ .../storage/aggregate/GroupByTransform.java | 158 ++++++++++++ 12 files changed, 1795 insertions(+), 2 deletions(-) diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/MemoryGridResource.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/MemoryGridResource.java index bc28cec023..57523f8747 100644 --- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/MemoryGridResource.java +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/MemoryGridResource.java @@ -40,7 +40,7 @@ import org.apache.sis.util.ArgumentChecks; * * @author Johann Sorel (Geomatys) * @author Martin Desruisseaux (Geomatys) - * @version 1.1 + * @version 1.3 * @since 1.1 * @module */ @@ -48,7 +48,7 @@ public class MemoryGridResource extends AbstractGridCoverageResource { /** * The grid coverage specified at construction time. */ - private final GridCoverage coverage; + public final GridCoverage coverage; /** * Creates a new coverage stored in memory. diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java new file mode 100644 index 0000000000..44faa761f6 --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.awt.image.RenderedImage; +import org.apache.sis.coverage.grid.GridExtent; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.coverage.grid.GridCoverage; +import org.apache.sis.coverage.SubspaceNotSpecifiedException; + + +/** + * A grid coverage where a single dimension is the concatenation of many grid coverages. + * All components must have the same "grid to CRS" transform, except for a translation term. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class ConcatenatedGridCoverage extends GridCoverage { + /** + * The slices of this coverage, in the same order than {@link #coordinatesOfSlices}. + * Each slice is not necessarily 1 cell tick; larger slices are accepted. + * The length of this array shall be at least 2. + */ + private final GridCoverage[] slices; + + /** + * The object for identifying indices in the {@link #slices} array. + */ + private final GridSliceLocator locator; + + /** + * Index of the first slice in {@link #locator}. + */ + private final int startAt; + + /** + * View over this grid coverage after conversion of sample values, or {@code null} if not yet created. + * May be {@code this} if we determined that there is no conversion or the conversion is identity. + * + * @see #forConvertedValues(boolean) + */ + private transient ConcatenatedGridCoverage convertedView; + + /** + * Creates a new aggregated coverage. + */ + ConcatenatedGridCoverage(final ConcatenatedGridResource source, final GridGeometry domain, + final GridCoverage[] slices, final int startAt) + { + super(domain, source.getSampleDimensions()); + this.slices = slices; + this.startAt = startAt; + this.locator = source.locator; + } + + /** + * Creates a new aggregated coverage for the result of a conversion from/to package values. + * This constructor assumes that all slices use the same sample dimensions. + */ + private ConcatenatedGridCoverage(final ConcatenatedGridCoverage source, final GridCoverage[] slices) { + super(source.getGridGeometry(), slices[0].getSampleDimensions()); + this.slices = slices; + this.startAt = source.startAt; + this.locator = source.locator; + convertedView = source; + } + + /** + * Returns a grid coverage that contains real values or sample values, + * depending if {@code converted} is {@code true} or {@code false} respectively. + * This method delegates to all slices in this concatenated coverage. + * + * @param converted {@code true} for a coverage containing converted values, + * or {@code false} for a coverage containing packed values. + * @return a coverage containing requested values. May be {@code this} but never {@code null}. + */ + @Override + public synchronized GridCoverage forConvertedValues(final boolean converted) { + if (convertedView == null) { + boolean changed = false; + final GridCoverage[] c = new GridCoverage[slices.length]; + for (int i=0; i<c.length; i++) { + final GridCoverage source = slices[i]; + changed |= (c[i] = source.forConvertedValues(converted)) != source; + } + convertedView = changed ? new ConcatenatedGridCoverage(this, c) : this; + } + return convertedView; + } + + /** + * Returns a two-dimensional slice of grid data as a rendered image. + * + * @param extent a subspace of this grid coverage extent where all dimensions except two have a size of 1 cell. + * @return the grid slice as a rendered image. Image location is relative to {@code sliceExtent}. + */ + @Override + public RenderedImage render(GridExtent extent) { + int lower = startAt, upper = lower + slices.length; + if (extent != null) { + upper = locator.getUpper(extent, lower, upper); + lower = locator.getLower(extent, lower, upper); + } else { + extent = gridGeometry.getExtent(); + } + if (upper - lower != 1) { + throw new SubspaceNotSpecifiedException(); + } + return slices[lower].render(locator.toSliceExtent(extent, lower)); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java new file mode 100644 index 0000000000..706f4f4c71 --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.Arrays; +import java.util.Optional; +import org.opengis.geometry.Envelope; +import org.opengis.referencing.operation.TransformException; +import org.apache.sis.coverage.SampleDimension; +import org.apache.sis.coverage.grid.GridCoverage; +import org.apache.sis.coverage.grid.GridDerivation; +import org.apache.sis.coverage.grid.GridExtent; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.coverage.grid.GridRoundingMode; +import org.apache.sis.geometry.ImmutableEnvelope; +import org.apache.sis.storage.AbstractGridCoverageResource; +import org.apache.sis.storage.DataStoreException; +import org.apache.sis.storage.GridCoverageResource; +import org.apache.sis.storage.RasterLoadingStrategy; +import org.apache.sis.storage.event.StoreListeners; +import org.apache.sis.internal.storage.MemoryGridResource; +import org.apache.sis.internal.util.UnmodifiableArrayList; +import org.apache.sis.internal.util.CollectionsExt; +import org.apache.sis.util.ArraysExt; + + +/** + * A grid coverage resource where a single dimension is the concatenation of many grid coverage resources. + * All components must have the same "grid to CRS" transform. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class ConcatenatedGridResource extends AbstractGridCoverageResource { + /** + * The grid geometry of this aggregated resource. + * + * @see #getGridGeometry() + */ + private final GridGeometry gridGeometry; + + /** + * The ranges of sample values of this aggregated resource. + * Shall be an unmodifiable list. + * + * @see #getSampleDimensions() + */ + private final List<SampleDimension> sampleDimensions; + + /** + * The slices of this resource, in the same order than {@link #coordinatesOfSlices}. + * Each slice is not necessarily 1 cell tick; larger slices are accepted. + */ + private final GridCoverageResource[] slices; + + /** + * The object for identifying indices in the {@link #slices} array. + */ + final GridSliceLocator locator; + + /** + * The envelope of this aggregate, or {@code null} if not yet computed. + * May also be {@code null} if no slice declare an envelope, or if the union can not be computed. + * + * @see #getEnvelope() + */ + private ImmutableEnvelope envelope; + + /** + * Whether {@link #envelope} has been initialized. + * The envelope may still be null if the initialization failed. + */ + private boolean envelopeIsEvaluated; + + /** + * The resolutions, or {@code null} if not yet computed. Can be an empty array after computation. + * + * @see #getResolutions() + */ + private double[][] resolutions; + + /** + * Creates a new aggregated resource. + * + * @param listeners listeners of the parent resource, or {@code null} if none. + * @param domain value to be returned by {@link #getGridGeometry()}. + * @param ranges value to be returned by {@link #getSampleDimensions()}. + * @param slices the slices of this resource, in the same order than {@code coordinatesOfSlices}. + */ + ConcatenatedGridResource(final StoreListeners listeners, + final GridGeometry domain, + final List<SampleDimension> ranges, + final GridCoverageResource[] slices, + final GridSliceLocator locator) + { + super(listeners, false); + this.gridGeometry = domain; + this.sampleDimensions = ranges; + this.slices = slices; + this.locator = locator; + } + + /** + * Returns the grid geometry of this aggregated resource. + */ + @Override + public final GridGeometry getGridGeometry() { + return gridGeometry; + } + + /** + * Returns the ranges of sample values of this aggregated resource. + */ + @Override + @SuppressWarnings("ReturnOfCollectionOrArrayField") + public final List<SampleDimension> getSampleDimensions() { + return sampleDimensions; + } + + /** + * Returns the spatiotemporal envelope of this resource. + * + * @return the spatiotemporal resource extent. + * @throws DataStoreException if an error occurred while reading or computing the envelope. + */ + @Override + public synchronized Optional<Envelope> getEnvelope() throws DataStoreException { + if (!envelopeIsEvaluated) { + try { + envelope = GroupAggregate.unionOfComponents(slices); + } catch (TransformException e) { + listeners.warning(e); + } + envelopeIsEvaluated = true; + } + return Optional.ofNullable(envelope); + } + + /** + * Returns the preferred resolutions (in units of CRS axes) for read operations in this data store. + * This method returns only the resolution that are declared by all coverages. + * + * @return preferred resolutions for read operations in this data store, or an empty array if none. + * @throws DataStoreException if an error occurred while reading definitions from the underlying data store. + */ + @Override + public synchronized List<double[]> getResolutions() throws DataStoreException { + double[][] common = resolutions; + if (common == null) { + int count = 0; + for (final GridCoverageResource slice : slices) { + final double[][] sr = CollectionsExt.toArray(slice.getResolutions(), double[].class); + if (sr != null) { // Should never be null, but we are paranoiac. + if (common == null) { + common = sr; + count = sr.length; + } else { + int retained = 0; + for (int i=0; i<count; i++) { + final double[] r = common[i]; + for (int j=0; j<sr.length; j++) { + if (Arrays.equals(r, sr[j])) { + common[retained++] = r; + sr[j] = null; + break; + } + } + } + count = retained; + if (count == 0) break; + } + } + } + resolutions = common = ArraysExt.resize(common, count); + } + return UnmodifiableArrayList.wrap(common); + } + + /** + * Returns an indication about when the "physical" loading of raster data will happen. + * This method returns the most conservative value of all slices. + * + * @return current raster data loading strategy for this resource. + * @throws DataStoreException if an error occurred while fetching data store configuration. + */ + @Override + public RasterLoadingStrategy getLoadingStrategy() throws DataStoreException { + RasterLoadingStrategy common = null; + for (final GridCoverageResource slice : slices) { + final RasterLoadingStrategy sr = slice.getLoadingStrategy(); + if (sr != null) { // Should never be null, but we are paranoiac. + if (common == null || sr.ordinal() < common.ordinal()) { + common = sr; + if (common.ordinal() == 0) { + break; + } + } + } + } + return common; + } + + /** + * Sets the preferred strategy about when to do the "physical" loading of raster data. + * Slices are free to replace the given strategy by another one. + * + * @param strategy the desired strategy for loading raster data. + * @return {@code true} if the given strategy has been accepted by at least one slice. + * @throws DataStoreException if an error occurred while setting data store configuration. + */ + @Override + public boolean setLoadingStrategy(final RasterLoadingStrategy strategy) throws DataStoreException { + boolean accepted = false; + for (final GridCoverageResource slice : slices) { + accepted |= slice.setLoadingStrategy(strategy); + } + return accepted; + } + + /** + * Loads a subset of the grid coverage represented by this resource. + * + * @param domain desired grid extent and resolution, or {@code null} for reading the whole domain. + * @param range 0-based indices of sample dimensions to read, or {@code null} or an empty sequence for reading them all. + * @return the grid coverage for the specified domain and range. + * @throws DataStoreException if an error occurred while reading the grid coverage data. + */ + @Override + public GridCoverage read(GridGeometry domain, final int... ranges) throws DataStoreException { + int lower = 0, upper = slices.length; + if (domain != null) { + final GridDerivation subgrid = gridGeometry.derive().rounding(GridRoundingMode.ENCLOSING).subgrid(domain); + domain = subgrid.build(); + final GridExtent sliceExtent = subgrid.getIntersection(); + upper = locator.getUpper(sliceExtent, lower, upper); + lower = locator.getLower(sliceExtent, lower, upper); + } + /* + * At this point we got the indices of the slices to read. The range should not be empty (upper > lower). + * Create arrays with only the requested range, without keeping reference to this concatenated resource, + * for allowing garbage-collection of resources outside that range. + */ + final GridCoverage[] coverages = new GridCoverage[upper - lower]; + for (int i=0; i < coverages.length; i++) { + final GridCoverageResource slice = slices[lower + i]; + if (slice instanceof MemoryGridResource) { + coverages[i] = ((MemoryGridResource) slice).coverage; + } else { + coverages[i] = slice.read(domain, ranges); + } + } + if (coverages.length == 1) { + return coverages[0]; + } + domain = locator.union(gridGeometry, Arrays.asList(coverages), (c) -> c.getGridGeometry().getExtent()); + return new ConcatenatedGridCoverage(this, domain, coverages, lower); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/CoverageAggregator.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/CoverageAggregator.java new file mode 100644 index 0000000000..ae72ea938b --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/CoverageAggregator.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.Locale; +import java.util.stream.Stream; +import org.opengis.referencing.operation.NoninvertibleTransformException; +import org.apache.sis.storage.Resource; +import org.apache.sis.storage.Aggregate; +import org.apache.sis.storage.DataStoreException; +import org.apache.sis.storage.DataStoreContentException; +import org.apache.sis.storage.GridCoverageResource; +import org.apache.sis.storage.event.StoreListeners; +import org.apache.sis.util.collection.BackingStoreException; + + +/** + * Creates a grid coverage resource from an aggregation of an arbitrary amount of other resources. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +public final class CoverageAggregator extends Group<GroupBySample> { + /** + * The listeners of the parent resource (typically a {@link org.apache.sis.storage.DataStore}), + * or {@code null} if none. + */ + private final StoreListeners listeners; + + /** + * Creates an initially empty aggregator. + * + * @param listeners listeners of the parent resource, or {@code null} if none. + * This is usually the listeners of the {@link org.apache.sis.storage.DataStore}. + */ + public CoverageAggregator(final StoreListeners listeners) { + this.listeners = listeners; + } + + /** + * Returns a name of the aggregate to be created. + * This is used only if this aggregator find resources having different sample dimensions. + * + * @param locale the locale for the name to return, or {@code null} for the default. + * @return a name which can be used as aggregation name, or {@code null} if none. + */ + @Override + final String getName(final Locale locale) { + return (listeners != null) ? listeners.getSourceName() : null; + } + + /** + * Adds all grid resources provided by the given stream. This method can be invoked from any thread. + * It delegates to {@link #add(GridCoverageResource)} for each element in the stream. + * + * @param resources resources to add. + * @throws DataStoreException if a resource can not be used. + */ + public void addAll(final Stream<? extends GridCoverageResource> resources) throws DataStoreException { + try { + resources.forEach((resource) -> { + try { + add(resource); + } catch (DataStoreException e) { + throw new BackingStoreException(e); + } + }); + } catch (BackingStoreException e) { + throw e.unwrapOrRethrow(DataStoreException.class); + } + } + + /** + * Adds the given resource. This method can be invoked from any thread. + * + * @param resource resource to add. + * @throws DataStoreException if the resource can not be used. + */ + public void add(final GridCoverageResource resource) throws DataStoreException { + final GroupBySample bySample = GroupBySample.getOrAdd(members, resource.getSampleDimensions()); + final GridSlice slice = new GridSlice(resource); + final List<GridSlice> slices; + try { + slices = slice.getList(bySample.members).members; + } catch (NoninvertibleTransformException e) { + throw new DataStoreContentException(e); + } + synchronized (slices) { + slices.add(slice); + } + } + + /** + * Builds a resource which is the aggregation or concatenation of all components added to this aggregator. + * The returned resource will be an instance of {@link GridCoverageResource} if possible, + * or an instance of {@link Aggregate} is some heterogeneity in grid geometries or sample dimensions + * prevent the concatenation of all coverages in a single resource. + * + * <p>This method is not thread safe. If the {@code add(…)} and {@code addAll(…)} methods were invoked + * in background threads, but all additions must be finished before this method is invoked.</p> + * + * @return the aggregation or concatenation of all components added to this aggregator. + */ + public Resource build() { + final GroupAggregate aggregate = prepareAggregate(listeners); + aggregate.fillWithChildAggregates(this, GroupBySample::createComponents); + return aggregate.simplify(); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/DimensionSelector.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/DimensionSelector.java new file mode 100644 index 0000000000..9ad0278236 --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/DimensionSelector.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.Arrays; +import java.math.BigInteger; +import org.apache.sis.internal.util.Strings; + + +/** + * A helper class for choosing the dimension on which to perform aggregation. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class DimensionSelector implements Comparable<DimensionSelector> { + /** + * The dimension examined by this selector. + */ + final int dimension; + + /** + * Grid coordinate in a single dimension of the grid extent, one value for each slice. + * It may be the grid low, mid or high value, it does not matter for this class as long + * as they are consistent. + */ + private final long[] positions; + + /** + * Sum of grid extent size of each slice. + * This is updated for each new slice added to this selector. + */ + private BigInteger sumOfSize; + + /** + * Increment in unit of the extent size. This calculation is based on mean values only. + * It is computed after the {@link #positions} array has been completed with data from all slices. + */ + private double relativeIncrement; + + /** + * Difference between minimal and maximal increment. + * This is computed after the {@link #positions} array has been completed with data from all slices. + */ + private long incrementRange; + + /** + * {@code true} if all {@link #positions} values are the same. + * This field is valid only after {@link #finish()} call. + */ + boolean isConstantPosition; + + /** + * Prepares a new selector for a single dimension. + * + * @param dim the dimension examined by this selector. + * @param n number of slices. + */ + DimensionSelector(final int dim, final int n) { + dimension = dim; + positions = new long[n]; + sumOfSize = BigInteger.ZERO; + } + + /** + * Sets the extent of a single slice. + * + * @param i index of the slice. + * @param pos position of the slice. Could be low, mid or high index, as long as the choice is kept consistent. + * @param size size of the extent, in number of cells. + */ + final void setSliceExtent(final int i, final long pos, final long size) { + positions[i] = pos; + sumOfSize = sumOfSize.add(BigInteger.valueOf(size)); + } + + /** + * Computes the {@linkplain #increment} between slices after all positions have been specified. + * This method is invoked in parallel (on different instances) for each dimension. + */ + final void finish() { + Arrays.sort(positions); // Not `parallelSort(…)` because this method is already invoked in parallel. + long maxInc = 0; + long minInc = Long.MAX_VALUE; + BigInteger sumOfInc = BigInteger.ZERO; + long previous = positions[0]; + for (int i=1; i<positions.length; i++) { + final long p = positions[i]; + final long d = p - previous; + if (d != 0) { + if (d < minInc) minInc = d; + if (d > maxInc) maxInc = d; + sumOfInc = sumOfInc.add(BigInteger.valueOf(d)); + previous = p; + } + } + isConstantPosition = (maxInc == 0); + if (minInc <= maxInc) { + relativeIncrement = sumOfInc.doubleValue() / sumOfSize.doubleValue(); + incrementRange = maxInc - minInc; // Can not overflow because minInc >= 0. + /* + * TODO: we may have a mosaic if `incrementRange == 0 && maxInc == size`. + * Or maybe we could accept `maxInc <= minSize`. + */ + } + } + + /** + * Compares for order of "probability" that a dimension is the one to aggregate. + * After using this comparator, dimensions that are more likely to be the ones to + * aggregate are sorted last. Because the order is defined that way, sorting the + * {@code DimensionSelector} array will have no effect in the most typical cases + * where the dimensions to aggregate are the last ones. + */ + @Override + public int compareTo(final DimensionSelector other) { + int c = Boolean.compare(other.isConstantPosition, isConstantPosition); // Non-constant sorted last. + if (c == 0) { + c = Double.compare(relativeIncrement, other.relativeIncrement); // Largest values sorted last. + if (c == 0) { + c = Long.compare(other.incrementRange, incrementRange); // Smallest values sorted last. + } + } + return c; + } + + /** + * Returns a string representation for debugging purposes. + */ + @Override + public String toString() { + return Strings.toString(getClass(), "dimension", dimension, "relativeIncrement", relativeIncrement); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GridSlice.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GridSlice.java new file mode 100644 index 0000000000..0c3e90dc4c --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GridSlice.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.Map; +import java.util.List; +import java.util.Arrays; +import org.opengis.referencing.datum.PixelInCell; +import org.opengis.referencing.operation.Matrix; +import org.opengis.referencing.operation.MathTransform; +import org.opengis.referencing.operation.NoninvertibleTransformException; +import org.apache.sis.referencing.operation.matrix.MatrixSIS; +import org.apache.sis.storage.GridCoverageResource; +import org.apache.sis.storage.DataStoreException; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.coverage.grid.GridExtent; +import org.apache.sis.internal.util.Numerics; +import org.apache.sis.internal.util.Strings; + + +/** + * A grid resource which is a slice in a larger coverage. + * A slice is not necessarily 1 cell tick; larger slices are accepted. + * + * <h2>Usage context</h2> + * Instances of {@code Gridslice} are grouped by CRS, then instances having the same CRS + * are grouped by "grid to CRS" transform in the {@link GroupByTransform#members} list. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class GridSlice { + /** + * The resource associated to this slice. + */ + final GridCoverageResource resource; + + /** + * Geometry of the grid coverage or resource. + */ + private final GridGeometry geometry; + + /** + * Translation from source coordinates of {@link GroupByTransform#gridToCRS} + * to grid coordinates of {@link #geometry}. Shall be considered read-only + * after initialization of {@link #setOffset(MatrixSIS)}. + */ + private final long[] offset; + + /** + * Creates a new slice for the specified resource. + * + * @param slice resource associated to this slice. + */ + GridSlice(final GridCoverageResource slice) throws DataStoreException { + resource = slice; + geometry = slice.getGridGeometry(); + offset = new long[geometry.getDimension()]; + } + + /** + * Sets the {@link #offset} terms to the values of the translation columns of the given matrix. + * This method shall be invoked if and only if {@link #isIntegerTranslation(Matrix)} returned {@code true}. + * + * @param groupToSlice conversion from source coordinates of {@link GroupByTransform#gridToCRS} + * to grid coordinates of {@link #geometry}. + * + * @see #getOffset(Map) + */ + private void setOffset(final MatrixSIS groupToSlice) { + final int i = groupToSlice.getNumCol() - 1; + for (int j=0; j<offset.length; j++) { + offset[j] = groupToSlice.getInteger(j, i); + } + } + + /** + * Returns {@code true} if the given matrix is the identity matrix except for translation terms. + * Translation terms must be integer values. + * + * @param groupToSlice conversion from {@link GroupByTransform#gridToCRS} + * source coordinates to {@link #gridToCRS} source coordinates. + * @return whether the matrix is identity, ignoring integer translation. + * + * @see org.apache.sis.referencing.operation.matrix.Matrices#isTranslation(Matrix) + */ + private static boolean isIntegerTranslation(final Matrix groupToSlice) { + final int numRows = groupToSlice.getNumRow(); + final int numCols = groupToSlice.getNumCol(); + for (int j=0; j<numRows; j++) { + for (int i=0; i<numCols; i++) { + double tolerance = Numerics.COMPARISON_THRESHOLD; + double e = groupToSlice.getElement(j, i); + if (i == j) { + e--; + } else if (i == numCols - 1) { + final double a = Math.abs(e); + if (a > 1) { + tolerance = Math.min(tolerance*a, 0.125); + } + e -= Math.rint(e); + } + if (!(Math.abs(e) <= tolerance)) { // Use `!` for catching NaN. + return false; + } + } + } + return true; + } + + /** + * Returns the group of objects associated to the CRS and "grid to CRS" transform. + * The CRS comparisons ignore metadata and transform comparisons ignore integer translations. + * This method takes a synchronization lock on the given list. + * + * @param groups the list where to search for a group. + * @return group of objects associated to the given transform (never null). + * @throws NoninvertibleTransformException if the transform is not invertible. + */ + final GroupByTransform getList(final List<GroupByCRS<GroupByTransform>> groups) + throws NoninvertibleTransformException + { + final MathTransform gridToCRS = geometry.getGridToCRS(PixelInCell.CELL_CORNER); + final MathTransform crsToGrid = gridToCRS.inverse(); + final List<GroupByTransform> transforms = GroupByCRS.getOrAdd(groups, geometry).members; + synchronized (transforms) { + for (final GroupByTransform c : transforms) { + final Matrix groupToSlice = c.linearTransform(crsToGrid); + if (groupToSlice != null && isIntegerTranslation(groupToSlice)) { + setOffset(MatrixSIS.castOrCopy(groupToSlice)); + return c; + } + } + final GroupByTransform c = new GroupByTransform(geometry, gridToCRS); + transforms.add(c); + return c; + } + } + + /** + * Returns the grid extent of this slice. The grid coordinate system is specific to this slice. + * For converting grid coordinates to the concatenated grid coordinate system, {@link #offset} + * must be subtracted. + */ + final GridExtent getGridExtent() { + return geometry.getExtent(); + } + + /** + * Writes information about grid extent into the given {@code DimensionSelector} objects. + * This is invoked by {@link GroupByTransform#findConcatenatedDimensions()} for choosing + * a dimension to concatenate. + */ + final void getGridExtent(final int i, final DimensionSelector[] writeTo) { + final GridExtent extent = getGridExtent(); + for (int dim = writeTo.length; --dim >= 0;) { + writeTo[dim].setSliceExtent(i, Math.subtractExact(extent.getMedian(dim), offset[dim]), extent.getSize(dim)); + } + } + + /** + * Returns the low grid index in the given dimension, relative to the grid of the group. + * This is invoked by {@link GroupByTransform#sortAndGetLows(int)} for sorting coverages. + * + * @param dim dimension of the desired grid coordinates. + * @return low index in the coordinate system of the group grid. + */ + final long getGridLow(final int dim) { + return Math.subtractExact(geometry.getExtent().getLow(dim), offset[dim]); + } + + /** + * Returns the translation from source coordinates of {@link GroupByTransform#gridToCRS} to + * grid coordinates of {@link #geometry}. This method returns a unique instance if possible. + * + * @param shared a pool of existing offset instances. + * @return translation from aggregated grid geometry to slice. Shall be considered read-only. + */ + final long[] getOffset(final Map<GridSlice,long[]> shared) { + final long[] old = shared.putIfAbsent(this, offset); + return (old != null) ? old : offset; + } + + /** + * Compares the offset of this grid slice with the offset of given slice. + * This method is defined only for the purpose of {@link #getOffset(Map)}. + * Equality should not be used in other context. + */ + @Override + public final boolean equals(final Object other) { + return (other instanceof GridSlice) && Arrays.equals(((GridSlice) other).offset, offset); + } + + /** + * Returns a hash code for the offset consistently with {@link #equals(Object)} purpose. + */ + @Override + public final int hashCode() { + return Arrays.hashCode(offset); + } + + /** + * Returns a string representation for debugging purposes. + */ + @Override + public String toString() { + Object id = null; + if (resource != null) try { + id = resource.getIdentifier().orElse(null); + } catch (DataStoreException e) { + id = e.toString(); + } + return Strings.toString(getClass(), null, id); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GridSliceLocator.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GridSliceLocator.java new file mode 100644 index 0000000000..2269efc19e --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GridSliceLocator.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.Map; +import java.util.List; +import java.util.Arrays; +import java.util.HashMap; +import java.util.function.Function; +import org.opengis.referencing.datum.PixelInCell; +import org.opengis.metadata.spatial.DimensionNameType; +import org.apache.sis.coverage.grid.GridExtent; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.storage.GridCoverageResource; + + +/** + * Coordinates of slices together with search methods. + * + * @todo Bilinear search needs to be replaced by an R-Tree. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class GridSliceLocator { + /** + * The dimension on which the searches are done. + * + * @todo If we replace bilinear search by an R-Tree, then it would be an array of all dimensions returned + * by {@code findConcatenatedDimensions()}. Only those dimensions need to be managed by an R-Tree. + * + * @see GroupByTransform#findConcatenatedDimensions() + */ + private final int searchDimension; + + /** + * Lows grid coordinates of each slice (inclusive) in the search dimension. + * Values must be sorted in increasing order. Duplicated values may exist. + * + * @todo Replace by an R-Tree. + */ + private final long[] sliceLows; + + /** + * Highs grid coordinates of each slice (inclusive) in the search dimension. + * Values are for slices in the same order as {@link #sliceLows}. + * This is not <strong>not</strong> guaranteed to be sorted in increasing order. + * + * @todo Replace by an R-Tree. + */ + private final long[] sliceHighs; + + /** + * Translation from source coordinates of {@link GroupByTransform#gridToCRS} to grid coordinates + * of {@link Gridslice#geometry}. Values are for slices in the same order as {@link #sliceLows}. + */ + private final long[][] offsets; + + /** + * Creates a new locator for slices at given coordinates. + * + * @param searchDimension the dimension on which the searches are done. + */ + GridSliceLocator(final List<GridSlice> slices, final int searchDimension, final GridCoverageResource[] resources) { + this.searchDimension = searchDimension; + + // TODO: use `parallelSort(…)` if https://bugs.openjdk.org/browse/JDK-8059093 is fixed. + slices.sort((o1, o2) -> Long.compare(o1.getGridLow(searchDimension), o2.getGridLow(searchDimension))); + + sliceLows = new long[resources.length]; + sliceHighs = new long[resources.length]; + offsets = new long[resources.length][]; + final Map<GridSlice,long[]> shared = new HashMap<>(); + for (int i=0; i<resources.length; i++) { + final GridSlice slice = slices.get(i); + final GridExtent extent = slice.getGridExtent(); + final long[] offset = slice.getOffset(shared); + final long dimOff = offset[searchDimension]; + sliceLows [i] = Math.subtractExact(extent.getLow (searchDimension), dimOff); + sliceHighs[i] = Math.subtractExact(extent.getHigh(searchDimension), dimOff); + resources [i] = slice.resource; + offsets [i] = offset; + } + } + + /** + * Creates a new grid geometry which is the union of all grid extent in the concatenated resource. + * + * @param <E> type of slice objects. + * @param base base geometry to expand. + * @param slices objects providing the grid extents. + * @param getter getter method for getting the grid extents from slices. + * @return expanded grid geometry. + */ + final <E> GridGeometry union(final GridGeometry base, final List<E> slices, final Function<E,GridExtent> getter) { + GridExtent extent = base.getExtent(); + final int dimension = extent.getDimension(); + final DimensionNameType[] axes = new DimensionNameType[dimension]; + final long[] low = new long[dimension]; + final long[] high = new long[dimension]; + for (int i=0; i<dimension; i++) { + axes[i] = extent.getAxisType(i).orElse(null); + low [i] = extent.getLow (i); + high[i] = extent.getHigh(i); + } + boolean changed = false; + final int count = slices.size(); + for (int i=0; i<count; i++) { + final GridExtent slice = getter.apply(slices.get(i)); + for (int j=0; j<dimension; j++) { + final long offset = offsets[i][j]; + long v; + if ((v = Math.subtractExact(slice.getLow (j), offset)) < low [j]) {low [j] = v; changed = true;} + if ((v = Math.subtractExact(slice.getHigh(j), offset)) > high[j]) {high[j] = v; changed = true;} + } + } + if (!changed) { + return base; + } + extent = new GridExtent(axes, low, high, true); + return new GridGeometry(extent, PixelInCell.CELL_CORNER, base.getGridToCRS(PixelInCell.CELL_CORNER), + base.isDefined(GridGeometry.CRS) ? base.getCoordinateReferenceSystem() : null); + } + + /** + * Returns the extent to use for querying a coverage from the slice at the given index. + * + * @param extent extent in units of aggregated grid coverage cells. + * @param slice index of the slice on which to delegate an operation. + * @return extent in units of the slice grid coverage. + */ + final GridExtent toSliceExtent(final GridExtent extent, final int slice) { + return extent.translate(offsets[slice]); + } + + /** + * Returns the index after the last slice which may intersect the given extent. + * + * @param sliceExtent the extent to search. + * @param fromIndex index of the first slice to include in the search. + * @param toIndex index after the last slice to include in the search. + */ + final int getUpper(final GridExtent sliceExtent, final int fromIndex, final int toIndex) { + final long high = sliceExtent.getHigh(searchDimension); + int upper = Arrays.binarySearch(sliceLows, fromIndex, toIndex, high); + if (upper < 0) { + upper = ~upper; // Index of first slice that can not intersect, because slice.low > high. + } else { + do upper++; + while (upper < toIndex && sliceLows[upper] <= high); + } + return upper; + } + + /** + * Returns the index of the fist slice which intersect the given extent. + * This method performs a linear search. For better performance, it should be invoked + * with {@code toIndex} parameter set to {@link #getUpper(GridExtent, int, int)} value. + * + * <h4>Limitations</h4> + * Current implementation assumes that {@link #sliceHighs} are sorted in increasing order, + * which is not guaranteed. For a robust search, we would need an R-Tree. + * + * @param sliceExtent the extent to search. + * @param fromIndex index of the first slice to include in the search. + * @param toIndex index after the last slice to include in the search. + */ + final int getLower(final GridExtent sliceExtent, final int fromIndex, int toIndex) { + final long low = sliceExtent.getLow(searchDimension); + while (toIndex > fromIndex) { + if (sliceHighs[--toIndex] < low) { + return toIndex + 1; + } + } + return toIndex; + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/Group.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/Group.java new file mode 100644 index 0000000000..2abf4bdfa4 --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/Group.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.ArrayList; +import java.util.Locale; +import java.util.stream.Stream; +import org.apache.sis.internal.util.Strings; +import org.apache.sis.storage.event.StoreListeners; + + +/** + * Base class for containers for a list of elements grouped by some attribute. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * + * @param <E> type of objects in this group. + * + * @since 1.3 + * @module + */ +abstract class Group<E> { + /** + * All members of this group. This list is populated by calls to {@link GridSlice#addTo(List)}. + * Accesses to this list should be synchronized during the phase when this list is populated, + * because that part may be parallelized by {@link CoverageAggregator#addResources(Stream)}. + * No synchronization is needed after. + */ + final List<E> members; + + /** + * Creates a new group of objects associated to some attribute defined by subclasses. + */ + Group() { + members = new ArrayList<>(); + } + + /** + * Returns a name for this group. + * This is used as the resource name if an aggregated resource needs to be created. + * + * @param locale the locale for the name to return, or {@code null} for the default. + * @return a name which can be used as aggregation name. + */ + abstract String getName(Locale locale); + + /** + * Prepares an initially empty aggregate. + * One of the {@code GroupAggregate.fill(…)} methods must be invoked after this method. + * + * @param listeners listeners of the parent resource, or {@code null} if none. + * @return an initially empty aggregate. + */ + final GroupAggregate prepareAggregate(final StoreListeners listeners) { + final int count = members.size(); + final String name; + if (count >= GroupAggregate.KEEP_ALIVE) { + name = getName(listeners == null ? null : listeners.getLocale()); + } else { + name = null; // Because it will not be needed. + } + return new GroupAggregate(listeners, name, count); + } + + /** + * Returns a string representation for debugging purposes. + */ + @Override + public String toString() { + final int count; + synchronized (members) { + count = members.size(); + } + return Strings.toString(getClass(), "name", getName(null), "count", count); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupAggregate.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupAggregate.java new file mode 100644 index 0000000000..0688d2d978 --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupAggregate.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.Collection; +import java.util.Optional; +import java.util.function.BiConsumer; +import org.opengis.geometry.Envelope; +import org.opengis.metadata.Metadata; +import org.opengis.referencing.operation.TransformException; +import org.apache.sis.storage.Resource; +import org.apache.sis.storage.Aggregate; +import org.apache.sis.storage.AbstractResource; +import org.apache.sis.storage.DataStoreException; +import org.apache.sis.storage.GridCoverageResource; +import org.apache.sis.storage.event.StoreListeners; +import org.apache.sis.internal.storage.MetadataBuilder; +import org.apache.sis.internal.util.UnmodifiableArrayList; +import org.apache.sis.coverage.SampleDimension; +import org.apache.sis.geometry.Envelopes; +import org.apache.sis.geometry.ImmutableEnvelope; + + +/** + * An aggregate created when, after grouping resources by CRS and other attributes, + * more than one group still exist. Those groups become components of an aggregate. + * This is used as temporary object during analysis, then kept alive in last resort + * when we can not build a single time series from a sequence of coverages at different times. + * + * <p>This class intentionally does not override {@link #getIdentifier()} because + * it would not be a persistent identifier.</p> + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class GroupAggregate extends AbstractResource implements Aggregate { + /** + * Minimum number of components for keeping this aggregate after analysis. + */ + static final int KEEP_ALIVE = 2; + + /** + * Name of this aggregate, or {@code null} if none. + * This is <strong>not</strong> a persistent identifier. + */ + private final String name; + + /** + * The components of this aggregate. Array elements are initially null, but should all become non-null + * after a {@code fill(…)} method has been invoked. If the length is smaller than {@value #KEEP_ALIVE}, + * then this aggregate is only a temporary object. + */ + private final Resource[] components; + + /** + * Whether all {@link #components} are {@link GridCoverageResource} elements. + * This is used for skipping calls to {@link #simplify()} when it is known that + * no component can be simplified. + */ + private boolean componentsAreCoverages; + + /** + * The envelope of this aggregate, or {@code null} if not yet computed. + * May also be {@code null} if no component declare an envelope, or if the union can not be computed. + * + * @see #getEnvelope() + */ + private ImmutableEnvelope envelope; + + /** + * Whether {@link #envelope} has been initialized. + * The envelope may still be null if the initialization failed. + */ + private boolean envelopeIsEvaluated; + + /** + * The sample dimensions of all children in this group, or an empty collection if they are not the same. + * This field is initially null, but should become non-null after a {@code fill(…)} method has been invoked. + * This is used for metadata only. + */ + List<SampleDimension> sampleDimensions; + + /** + * Creates a new aggregate with the specified number of components. + * One of the {@code fill(…)} methods must be invoked after this constructor. + * + * @param listeners listeners of the parent resource, or {@code null} if none. + * @param name name of this aggregate, or {@code null} if none. + * @param count expected number of components. + * + * @see Group#prepareAggregate(StoreListeners) + */ + GroupAggregate(final StoreListeners listeners, final String name, final int count) { + super(listeners, count < KEEP_ALIVE); + components = new Resource[count]; + this.name = name; + } + + /** + * Sets all components of this aggregate to sub-aggregates, which are themselves initialized with the given filler. + * This method may be invoked recursively if the sub-aggregates themselves have sub-sub-aggregates. + * + * @param <E> type of object in the group. + * @param children data for creating children, as one sub-aggregate per member of the {@code children} group. + * @param childFiller the action to execute for initializing each sub-aggregate. + * The first {@link BiConsumer} argument is a {@code children} member (the source) + * and the second argument is the sub-aggregate to initialize (the target). + */ + final <E> void fillWithChildAggregates(final Group<E> children, final BiConsumer<E,GroupAggregate> childFiller) { + for (int i=0; i < components.length; i++) { + final GroupAggregate child = children.prepareAggregate(listeners); + childFiller.accept(children.members.get(i), child); + components[i] = child; + } + } + + /** + * Sets all components of this aggregate to grid coverage resources. + * Children created by this method are leaf nodes. + * + * @param children date for creating children, as one coverage per member of the {@code children} group. + * @param ranges sample dimensions of the coverage to create. Stored as-is (not copied). + */ + @SuppressWarnings("AssignmentToCollectionOrArrayFieldFromParameter") // Copy done by GroupBySample constructor. + final void fillWithCoverageComponents(final List<GroupByTransform> children, final List<SampleDimension> ranges) { + componentsAreCoverages = true; + for (int i=0; i < components.length; i++) { + components[i] = children.get(i).createResource(listeners, ranges); + } + } + + /** + * Simplifies the resource tree by removing all aggregates of 1 component. + * + * @return the resource to use after simplification. + */ + final Resource simplify() { + if (!componentsAreCoverages) { + for (int i=0; i < components.length; i++) { + final Resource r = components[i]; + if (r instanceof GroupAggregate) { + components[i] = ((GroupAggregate) r).simplify(); + } + } + } + if (components.length == 1) { + return components[0]; + } + return this; + } + + /** + * Returns the components of this aggregate. + */ + @Override + public Collection<Resource> components() { + return UnmodifiableArrayList.wrap(components); + } + + /** + * Returns the spatiotemporal envelope of this resource. + * + * @return the spatiotemporal resource extent. + * @throws DataStoreException if an error occurred while reading or computing the envelope. + */ + @Override + public synchronized Optional<Envelope> getEnvelope() throws DataStoreException { + if (!envelopeIsEvaluated) { + try { + envelope = unionOfComponents(components); + } catch (TransformException e) { + listeners.warning(e); + } + envelopeIsEvaluated = true; + } + return Optional.ofNullable(envelope); + } + + /** + * Computes the union of envelopes provided by all the given resources. + * + * @param components the components for which to extract the envelope. + * @return union of envelope of all components, or {@code null} if none. + */ + static ImmutableEnvelope unionOfComponents(final Resource[] components) + throws DataStoreException, TransformException + { + final Envelope[] envelopes = new Envelope[components.length]; + for (int i=0; i < components.length; i++) { + final Resource r = components[i]; + if (r instanceof AbstractResource) { + envelopes[i] = ((AbstractResource) r).getEnvelope().orElse(null); + } else if (r instanceof GridCoverageResource) { + envelopes[i] = ((GridCoverageResource) r).getEnvelope().orElse(null); + } + } + return ImmutableEnvelope.castOrCopy(Envelopes.union(envelopes)); + } + + /** + * Creates when first requested the metadata about this aggregate. + * The metadata contains the title for this aggregation, the sample dimensions + * (if they are the same for all children) and the geographic bounding box. + */ + @Override + protected Metadata createMetadata() throws DataStoreException { + final MetadataBuilder builder = new MetadataBuilder(); + builder.addTitle(name); + try { + builder.addExtent(envelope); + } catch (TransformException e) { + listeners.warning(e); + } + if (sampleDimensions != null) { + for (final SampleDimension band : sampleDimensions) { + builder.addNewBand(band); + } + } + return builder.build(); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupByCRS.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupByCRS.java new file mode 100644 index 0000000000..ad7c393642 --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupByCRS.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.Locale; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.apache.sis.referencing.IdentifiedObjects; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.util.Utilities; + + +/** + * A container for a list of elements grouped by their CRS. The CRS comparisons ignore metadata. + * + * <h2>Usage for coverage aggregation</h2> + * {@code GroupByCRS} contains an arbitrary amount of {@link GroupByTransform} instances, + * which in turn contain an arbitrary amount of {@link GridSlice} instances. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * + * @param <E> type of objects in this group. + * + * @since 1.3 + * @module + */ +final class GroupByCRS<E> extends Group<E> { + /** + * The coordinate reference system of this group, or {@code null}. + * All {@linkplain #members} of this group use this CRS, + * possibly with ignorable differences in metadata. + */ + private final CoordinateReferenceSystem crs; + + /** + * Creates a new group of objects associated to the given CRS. + * + * @param crs coordinate reference system of this group, or {@code null}. + */ + private GroupByCRS(final CoordinateReferenceSystem crs) { + this.crs = crs; + } + + /** + * Returns a name for this group. + */ + @Override + final String getName(final Locale locale) { + return IdentifiedObjects.getDisplayName(crs, locale); + } + + /** + * Returns the group of objects associated to the given grid geometry. + * The CRS comparisons ignore metadata. + * This method takes a synchronization lock on the given list. + * + * @param <E> type of objects in groups. + * @param groups the list where to search for a group. + * @param geometry geometry of the grid coverage or resource. + * @return group of objects associated to the given CRS (never null). + */ + static <E> GroupByCRS<E> getOrAdd(final List<GroupByCRS<E>> groups, final GridGeometry geometry) { + return getOrAdd(groups, geometry.isDefined(GridGeometry.CRS) ? geometry.getCoordinateReferenceSystem() : null); + } + + /** + * Returns the group of objects associated to the given CRS. + * The CRS comparisons ignore metadata. + * This method takes a synchronization lock on the given list. + * + * @param <E> type of objects in groups. + * @param groups the list where to search for a group. + * @param crs coordinate reference system of the desired group, or {@code null}. + * @return group of objects associated to the given CRS (never null). + */ + private static <E> GroupByCRS<E> getOrAdd(final List<GroupByCRS<E>> groups, final CoordinateReferenceSystem crs) { + synchronized (groups) { + for (final GroupByCRS<E> c : groups) { + if (Utilities.equalsIgnoreMetadata(crs, c.crs)) { + return c; + } + } + final GroupByCRS<E> c = new GroupByCRS<>(crs); + groups.add(c); + return c; + } + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupBySample.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupBySample.java new file mode 100644 index 0000000000..1d2d5713ae --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupBySample.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.Locale; +import java.util.StringJoiner; +import org.apache.sis.coverage.SampleDimension; + + +/** + * A container for a list of elements grouped by their sample dimensions. + * + * <h2>Usage for coverage aggregation</h2> + * {@code GroupBySample} contains an arbitrary amount of {@link GroupByCRS} instances, + * which in turn contain an arbitrary amount of {@link GroupByTransform} instances, + * which in turn contain an arbitrary amount of {@link GridSlice} instances. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class GroupBySample extends Group<GroupByCRS<GroupByTransform>> { + /** + * The sample dimensions of this group. + */ + private final List<SampleDimension> ranges; + + /** + * Creates a new group of objects associated to the list of sample dimensions. + * + * @param ranges the sample dimensions of this group. + */ + private GroupBySample(final List<SampleDimension> ranges) { + this.ranges = ranges; // TODO: use List.copyOf(…) with JDK10. + } + + /** + * Returns a name for this group. + */ + @Override + final String getName(final Locale locale) { + final StringJoiner name = new StringJoiner(", "); + for (final SampleDimension range : ranges) { + name.add(range.getName().toInternationalString().toString(locale)); + } + return name.toString(); + } + + /** + * Returns the group of objects associated to the given ranges. + * This method takes a synchronization lock on the given list. + * + * @param <E> type of objects in groups. + * @param groups the list where to search for a group. + * @param ranges sample dimensions of the desired group. + * @return group of objects associated to the given ranges (never null). + */ + static GroupBySample getOrAdd(final List<GroupBySample> groups, final List<SampleDimension> ranges) { + synchronized (groups) { + for (final GroupBySample c : groups) { + if (ranges.equals(c.ranges)) { + return c; + } + } + final GroupBySample c = new GroupBySample(ranges); + groups.add(c); + return c; + } + } + + /** + * Creates sub-aggregates for each member of this group and add them to the given aggregate. + * + * @param destination where to add sub-aggregates. + */ + final void createComponents(final GroupAggregate destination) { + destination.sampleDimensions = ranges; + destination.fillWithChildAggregates(this, (byCRS,child) -> { + child.fillWithCoverageComponents(byCRS.members, ranges); + child.sampleDimensions = ranges; + }); + } +} diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupByTransform.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupByTransform.java new file mode 100644 index 0000000000..77257a619b --- /dev/null +++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/GroupByTransform.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.internal.storage.aggregate; + +import java.util.List; +import java.util.Arrays; +import java.util.Locale; +import java.text.NumberFormat; +import java.text.FieldPosition; +import org.opengis.referencing.operation.Matrix; +import org.opengis.referencing.operation.MathTransform; +import org.apache.sis.referencing.operation.transform.MathTransforms; +import org.apache.sis.storage.GridCoverageResource; +import org.apache.sis.storage.event.StoreListeners; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.coverage.SampleDimension; +import org.apache.sis.math.DecimalFunctions; +import org.apache.sis.util.resources.Vocabulary; +import org.apache.sis.util.ArraysExt; + + +/** + * A container for a list of slices grouped by their "grid to CRS" transform, ignoring integer translations. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.3 + * @since 1.3 + * @module + */ +final class GroupByTransform extends Group<GridSlice> { + /** + * Geometry of the grid coverage or resource. This is copied from the first + * {@link GridSlice#geometry} found in iteration order for this group. + * It may be a somewhat random grid geometry, unless {@link GridSlice} + * instances are sorted before processing. + */ + private final GridGeometry geometry; + + /** + * Value or {@code geometry.getGridToCRS(PixelInCell.CELL_CORNER)}. + * All {@linkplain #members} of this group use this transform, + * possibly with integer differences in translation terms. + */ + private final MathTransform gridToCRS; + + /** + * Creates a new group of objects associated to the given transform. + * + * @param geometry geometry of the grid coverage or resource. + * @param gridToCRS value or {@code geometry.getGridToCRS(PixelInCell.CELL_CORNER)}. + */ + GroupByTransform(final GridGeometry geometry, final MathTransform gridToCRS) { + this.geometry = geometry; + this.gridToCRS = gridToCRS; + } + + /** + * Returns a name for this group. + * This is used as the resource name if an aggregated resource needs to be created. + * Current implementation assumes that the main reason why many groups may exist is + * that they differ by their resolution. + */ + @Override + final String getName(final Locale locale) { + final Vocabulary v = Vocabulary.getResources(locale); + final StringBuffer b = new StringBuffer(v.getLabel(Vocabulary.Keys.Resolution)); + final NumberFormat f = NumberFormat.getIntegerInstance(v.getLocale()); + final FieldPosition p = new FieldPosition(0); + String separator = ""; + for (final double r : geometry.getResolution(true)) { // Should not fail when `gridToCRS` exists. + f.setMaximumFractionDigits(Math.max(DecimalFunctions.fractionDigitsForDelta(r / 100, false), 0)); + f.format(r, b.append(separator), p); + separator = " × "; + } + return b.toString(); + } + + /** + * Returns the conversion of pixel coordinates from this group to a slice if that conversion is linear. + * + * @param crsToGrid the "CRS to slice" transform. + * @return the conversion as an affine transform, or {@code null} if null or if the conversion is non-linear. + */ + final Matrix linearTransform(final MathTransform crsToGrid) { + if (gridToCRS.getTargetDimensions() == crsToGrid.getSourceDimensions()) { + return MathTransforms.getMatrix(MathTransforms.concatenate(gridToCRS, crsToGrid)); + } + return null; + } + + /** + * Returns dimensions to aggregate, in order of recommendation. + * Aggregations should use the first dimension in the returned list. + * + * @todo A future version should add {@code findMosaicDimensions()}, which should be tested first. + */ + private int[] findConcatenatedDimensions() { + final DimensionSelector[] selects; + synchronized (members) { // Should no longer be needed at this step, but we are paranoiac. + int i = members.size(); + if (i < 2) return ArraysExt.EMPTY_INT; + selects = new DimensionSelector[geometry.getDimension()]; + for (int dim = selects.length; --dim >= 0;) { + selects[dim] = new DimensionSelector(dim, i); + } + while (--i >= 0) { + members.get(i).getGridExtent(i, selects); + } + } + Arrays.stream(selects).parallel().forEach(DimensionSelector::finish); + Arrays.sort(selects); // Contains usually less than 5 elements. + final int[] dimensions = new int[selects.length]; + int count = 0; + for (int i=selects.length; --i >= 0;) { + if (selects[i].isConstantPosition) break; + dimensions[count++] = selects[i].dimension; + } + return ArraysExt.resize(dimensions, count); + } + + /** + * Sorts the slices in increasing order of low grid coordinates in the concatenated dimension. + * Then build a concatenated grid coverage resource capable to perform binary searches along that dimension. + * + * @param parentListeners listeners of the parent resource, or {@code null} if none. + * @param sampleDimensions the sample dimensions of the resource to build. + */ + final GridCoverageResource createResource(final StoreListeners parentListeners, final List<SampleDimension> ranges) { + final int[] dimensions = findConcatenatedDimensions(); + if (dimensions.length == 0) { + return null; + } + final int dim = dimensions[0]; + final GridSliceLocator locator; + final GridCoverageResource[] slices; + final GridGeometry domain; + synchronized (members) { // Should no longer be needed at this step, but we are paranoiac. + slices = new GridCoverageResource[members.size()]; + locator = new GridSliceLocator(members, dim, slices); + domain = locator.union(geometry, members, GridSlice::getGridExtent); + } + return new ConcatenatedGridResource(parentListeners, domain, ranges, slices, locator); + } +}