This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
The following commit(s) were added to refs/heads/geoapi-4.0 by this push:
new e16859b Read all tiles that are inside the requested domain at
`GridCoverageResource.read(…)` invocation time by default, but provide a
`RasterLoadingStrategy` enumeration for allowing developers to keep the
previous mode if desired. This enumeration is not in public API for now; we
will see how it works in practice first.
e16859b is described below
commit e16859b2c048227c88a16a045f63aba23d663d0d
Author: Martin Desruisseaux <martin.desruisse...@geomatys.com>
AuthorDate: Fri Jul 30 18:26:49 2021 +0200
Read all tiles that are inside the requested domain at
`GridCoverageResource.read(…)` invocation time by default,
but provide a `RasterLoadingStrategy` enumeration for allowing developers
to keep the previous mode if desired.
This enumeration is not in public API for now; we will see how it works in
practice first.
---
.../apache/sis/coverage/grid/GridCoverage2D.java | 1 -
.../org/apache/sis/storage/geotiff/DataCube.java | 14 ++-
.../org/apache/sis/storage/geotiff/DataSubset.java | 2 +-
.../sis/storage/geotiff/ImageFileDirectory.java | 4 +-
.../sis/internal/storage/AbstractGridResource.java | 28 +++++
.../internal/storage/RasterLoadingStrategy.java | 112 +++++++++++++++++++
.../sis/internal/storage/TiledGridResource.java | 121 ++++++++++++++++++++-
.../sis/test/storage/CoverageReadConsistency.java | 13 +++
8 files changed, 287 insertions(+), 8 deletions(-)
diff --git
a/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
b/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
index e1bc00f..837a475 100644
---
a/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
+++
b/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
@@ -608,7 +608,6 @@ public class GridCoverage2D extends GridCoverage {
* may force data loading earlier than desired.
*/
final ReshapedImage result = new ReshapedImage(data, xmin, ymin,
xmax, ymax);
- String error; assert (error = result.verify()) == null : error;
return result.isIdentity() ? data : result;
} catch (ArithmeticException e) {
throw new CannotEvaluateException(e.getMessage(), e);
diff --git
a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
index 23f2e66..1ecb856 100644
---
a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
+++
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
@@ -19,6 +19,7 @@ package org.apache.sis.storage.geotiff;
import java.nio.file.Path;
import java.awt.image.SampleModel;
import java.awt.image.BandedSampleModel;
+import org.apache.sis.storage.DataStore;
import org.apache.sis.storage.DataStoreException;
import org.apache.sis.storage.DataStoreContentException;
import org.apache.sis.coverage.grid.GridCoverage;
@@ -59,6 +60,14 @@ abstract class DataCube extends TiledGridResource implements
ResourceOnFileSyste
}
/**
+ * Returns the object on which to perform all synchronizations for
thread-safety.
+ */
+ @Override
+ protected final DataStore getSynchronizationLock() {
+ return reader.store;
+ }
+
+ /**
* Shortcut for a frequently requested information.
*/
final String filename() {
@@ -122,9 +131,9 @@ abstract class DataCube extends TiledGridResource
implements ResourceOnFileSyste
@Override
public final GridCoverage read(final GridGeometry domain, final int...
range) throws DataStoreException {
final long startTime = System.nanoTime();
- final GridCoverage coverage;
+ GridCoverage coverage;
try {
- synchronized (reader.store) {
+ synchronized (getSynchronizationLock()) {
final Subset subset = new Subset(domain, range, false);
final Compression compression = getCompression();
if (compression == null) {
@@ -145,6 +154,7 @@ abstract class DataCube extends TiledGridResource
implements ResourceOnFileSyste
Resources.Keys.UnsupportedCompressionMethod_1,
compression));
}
}
+ coverage = preload(coverage);
}
} catch (RuntimeException e) {
throw new
DataStoreException(reader.errors().getString(Errors.Keys.CanNotRead_1,
filename()), e);
diff --git
a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
index 313b262..d0a7574 100644
---
a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
+++
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
@@ -283,7 +283,7 @@ class DataSubset extends TiledGridCoverage implements
Localized {
final Tile[] missings = new Tile[iterator.tileCountInQuery];
int numMissings = 0;
boolean needsCompaction = false;
- synchronized (reader().store) {
+ synchronized (source.getSynchronizationLock()) {
do {
final WritableRaster tile = iterator.getCachedTile();
if (tile != null) {
diff --git
a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
index 778c3f5..a14997b 100644
---
a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
+++
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
@@ -1297,7 +1297,7 @@ final class ImageFileDirectory extends DataCube {
*/
@Override
public GridGeometry getGridGeometry() throws DataStoreContentException {
- synchronized (reader.store) {
+ synchronized (getSynchronizationLock()) {
if (referencing != null) {
GridGeometry gridGeometry = referencing.gridGeometry;
if (gridGeometry == null) try {
@@ -1321,7 +1321,7 @@ final class ImageFileDirectory extends DataCube {
@Override
@SuppressWarnings("ReturnOfCollectionOrArrayField")
public List<SampleDimension> getSampleDimensions() throws
DataStoreContentException {
- synchronized (reader.store) {
+ synchronized (getSynchronizationLock()) {
if (sampleDimensions == null) {
final SampleDimension[] dimensions = new
SampleDimension[samplesPerPixel];
final SampleDimension.Builder builder = new
SampleDimension.Builder();
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
index d502e8c..fe5b1ce 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
@@ -482,6 +482,34 @@ public abstract class AbstractGridResource extends
AbstractResource implements G
}
/**
+ * Returns an indication about when the "physical" loading of raster data
will happen.
+ * This is the strategy actually applied by this resource implementation,
not necessarily
+ * the strategy given in the last call to {@link
#setLoadingStrategy(RasterLoadingStrategy)
+ * setLoadingStrategy(…)}.
+ *
+ * <p>The default strategy is to load raster data at {@link #read read(…)}
method invocation time.</p>
+ *
+ * @return current raster data loading strategy for this resource.
+ */
+ public RasterLoadingStrategy getLoadingStrategy() {
+ return RasterLoadingStrategy.AT_READ_TIME;
+ }
+
+ /**
+ * Sets the preferred strategy about when to do the "physical" loading of
raster data.
+ * Implementations are free to ignore this parameter or to replace the
given strategy
+ * by the closest alternative that this resource can support.
+ *
+ * @param strategy the desired strategy for loading raster data.
+ * @return {@code true} if the given strategy has been accepted, or {@code
false}
+ * if this implementation replaced the given strategy by an
alternative.
+ */
+ public boolean setLoadingStrategy(final RasterLoadingStrategy strategy) {
+ ArgumentChecks.ensureNonNull("strategy", strategy);
+ return strategy == getLoadingStrategy();
+ }
+
+ /**
* Logs the execution of a {@link #read(GridGeometry, int...)} operation.
* The log level will be {@link Level#FINE} if the operation was quick
enough,
* or {@link PerformanceLevel#SLOW} or higher level otherwise.
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/RasterLoadingStrategy.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/RasterLoadingStrategy.java
new file mode 100644
index 0000000..7d674c2
--- /dev/null
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/RasterLoadingStrategy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.storage;
+
+import java.lang.ref.SoftReference;
+import java.awt.image.RenderedImage;
+import java.awt.image.ImagingOpException;
+import org.opengis.coverage.CannotEvaluateException;
+import org.apache.sis.coverage.grid.GridExtent;
+import org.apache.sis.coverage.grid.GridGeometry;
+import org.apache.sis.coverage.grid.GridCoverage;
+import org.apache.sis.storage.GridCoverageResource;
+import org.apache.sis.storage.DataStoreException;
+
+
+/**
+ * Time when the "physical" loading of raster data should happen. Some
resource implementations may
+ * not load data immediately when the {@linkplain GridCoverageResource#read
read method} is invoked,
+ * but instead defer the actual loading until the {@linkplain
GridCoverage#render image is rendered}.
+ * This enumeration gives some control over the time when data loading happens.
+ * The different strategies are compromises between memory consumption,
+ * redundant loading of same data and early error detection.
+ *
+ * <p>Enumeration values are ordered from the most eager strategy to the
laziest strategy.
+ * The eager strategy is fastest when all pixels are used, at the cost of
largest memory consumption.
+ * The lazy strategy is more efficient when only a few tiles will be used and
those tiles are not known in advance.
+ * The lazy strategy is also the only applicable one if the image is too large
for holding in memory.</p>
+ *
+ * @todo This enumeration is internal for now, but may move to public API in a
future version.
+ *
+ * @author Martin Desruisseaux (Geomatys)
+ * @version 1.1
+ * @since 1.1
+ * @module
+ */
+public enum RasterLoadingStrategy {
+ /**
+ * Raster data are loaded at {@link
GridCoverageResource#read(GridGeometry, int...)} invocation time.
+ * This is the most eager loading strategy.
+ *
+ * <p><b>Advantages:</b></p>
+ * <ul>
+ * <li>Fastest loading strategy if all pixels in the returned {@link
GridCoverage} will be used.</li>
+ * <li>No redundant data loading: all data stay in memory until the
resource is garbage-collected.</li>
+ * <li>Immediate error notification with a checked {@link
DataStoreException}.</li>
+ * </ul>
+ *
+ * <p><b>Disadvantages:</b></p>
+ * <ul>
+ * <li>Slower than other strategies if only a subset of the returned
{@link GridCoverage} will be used.</li>
+ * <li>Consume memory for the full {@link GridCoverage} as long as the
resource is referenced.</li>
+ * </ul>
+ */
+ AT_READ_TIME,
+
+ /**
+ * Raster data are loaded at {@link GridCoverage#render(GridExtent)}
invocation time.
+ * Speed and memory usage are at an intermediate level between {@link
#AT_READ_TIME} and {@link #AT_GET_TILE_TIME}.
+ *
+ * <p><b>Advantages:</b></p>
+ * <ul>
+ * <li>Faster than {@link #AT_GET_TILE_TIME} if all pixels in the
returned {@link RenderedImage} will be used.</li>
+ * <li>Load less data than {@link #AT_READ_TIME} if the {@link
GridExtent} is a sub-region of the coverage.</li>
+ * <li>Memory is released sooner (when the {@link RenderedImage} is
garbage-collected) than above mode.</li>
+ * </ul>
+ *
+ * <p><b>Disadvantages:</b></p>
+ * <ul>
+ * <li>Slower than {@link #AT_GET_TILE_TIME} if only a few tiles will be
used.</li>
+ * <li>Consume memory for the full {@link RenderedImage} as long as the
image is referenced.</li>
+ * <li>May reload the same data many times if images are discarded and
recreated.</li>
+ * <li>Unchecked {@link CannotEvaluateException} can happen relatively
late
+ * (at {@code render(…)} invocation time) in a chain of coverage
operations.</li>
+ * </ul>
+ */
+ AT_RENDER_TIME,
+
+ /**
+ * Raster data are loaded at {@link RenderedImage#getTile(int, int)}
invocation time.
+ * This is the laziest loading strategy.
+ * This is also the only strategy that can handle very large {@link
RenderedImage}s.
+ *
+ * <p><b>Advantages:</b></p>
+ * <ul>
+ * <li>Only the tiles that are actually used are loaded.</li>
+ * <li>Memory can be released at any time (tiles are kept by {@linkplain
SoftReference soft references}).</li>
+ * </ul>
+ *
+ * <p><b>Disadvantages:</b></p>
+ * <ul>
+ * <li>Slower read operations (numerous seeks when tiles are read in
random order).</li>
+ * <li>May reload the same data many times if tiles are discarded and
recreated.</li>
+ * <li>Unchecked {@link ImagingOpException} can happen late
+ * (at {@code getTile(…)} invocation time) in a chain of image
operations.</li>
+ * </ul>
+ */
+ AT_GET_TILE_TIME
+}
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
index e21c73e..212c1b4 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
@@ -20,13 +20,18 @@ import java.util.List;
import java.util.Arrays;
import java.awt.image.ColorModel;
import java.awt.image.SampleModel;
+import java.awt.image.RenderedImage;
import java.awt.image.WritableRaster;
import java.awt.image.RasterFormatException;
+import org.opengis.coverage.CannotEvaluateException;
import org.apache.sis.coverage.SampleDimension;
+import org.apache.sis.coverage.grid.GridCoverage;
+import org.apache.sis.coverage.grid.GridCoverage2D;
import org.apache.sis.coverage.grid.GridDerivation;
import org.apache.sis.coverage.grid.GridExtent;
import org.apache.sis.coverage.grid.GridGeometry;
import org.apache.sis.coverage.grid.GridRoundingMode;
+import org.apache.sis.storage.DataStore;
import org.apache.sis.storage.DataStoreException;
import org.apache.sis.storage.event.StoreListeners;
import org.apache.sis.util.collection.WeakValueHashMap;
@@ -96,6 +101,15 @@ public abstract class TiledGridResource extends
AbstractGridResource {
private final WeakValueHashMap<CacheKey, WritableRaster> rasters;
/**
+ * Whether all tiles should be loaded at {@code read(…)} method call or
deferred to a later time.
+ * This field is initially {@code null} and is initialized to its default
value only when needed.
+ *
+ * @see #getLoadingStrategy()
+ * @see #setLoadingStrategy(RasterLoadingStrategy)
+ */
+ private RasterLoadingStrategy loadingStrategy;
+
+ /**
* Creates a new resource.
*
* @param parent listeners of the parent resource, or {@code null} if
none.
@@ -106,13 +120,21 @@ public abstract class TiledGridResource extends
AbstractGridResource {
}
/**
+ * Returns the object on which to perform all synchronizations for
thread-safety.
+ * In current implementation, this is the {@link DataStore} that created
this resource.
+ * However this restriction may be relaxed in any future version.
+ *
+ * @return the synchronization lock.
+ */
+ protected abstract DataStore getSynchronizationLock();
+
+ /**
* Returns the size of tiles in this resource.
* The length of the returned array is the number of dimensions.
*
* @return the size of tiles in this resource.
- * @throws DataStoreException if an error occurred while fetching tile
size information.
*/
- protected abstract int[] getTileSize() throws DataStoreException;
+ protected abstract int[] getTileSize();
/**
* Returns the Java2D sample model describing pixel type and layout for
all bands.
@@ -329,4 +351,99 @@ public abstract class TiledGridResource extends
AbstractGridResource {
return selectedBands != null;
}
}
+
+ /**
+ * If the loading strategy is to load all tiles at {@code read(…)} time,
replaces the given coverage
+ * by a coverage will all data in memory. This method should be invoked by
subclasses at the end of
+ * their {@link #read(GridGeometry, int...)} method implementation.
+ *
+ * @param coverage the {@link TiledGridCoverage} to potentially replace
by a coverage with preloaded data.
+ * @return a coverage with preloaded data, or the given coverage if
preloading is not enabled.
+ * @throws DataStoreException if an error occurred while preloading data.
+ */
+ protected final GridCoverage preload(final GridCoverage coverage) throws
DataStoreException {
+ assert Thread.holdsLock(getSynchronizationLock());
+ // Note: `loadingStrategy` may still be null if unitialized.
+ if (loadingStrategy != RasterLoadingStrategy.AT_RENDER_TIME) {
+ /*
+ * In theory the following condition is redundant with
`supportImmediateLoading()`.
+ * We apply it anyway in case the coverage geometry is not what
was announced.
+ * This condition is also necessary if `loadingStrategy` has not
been initialized.
+ */
+ if (coverage.getGridGeometry().getDimension() ==
TiledGridCoverage.BIDIMENSIONAL) try {
+ final RenderedImage image = coverage.render(null);
+ return new GridCoverage2D(coverage.getGridGeometry(),
coverage.getSampleDimensions(), image);
+ } catch (RuntimeException e) {
+ /*
+ * The `coverage.render(…)` implementation may have wrapped
the checked `DataStoreException`
+ * because of API restriction. In that case we can unwrap the
exception here since this API
+ * allows it. This is one of the reasons for preferring the
`AT_READ_TIME` loading mode.
+ */
+ Throwable cause = e.getCause();
+ if (cause instanceof DataStoreException) {
+ throw (DataStoreException) cause;
+ }
+ /*
+ * The `CannotEvaluateException` wrapper is created by
`TiledGridCoverage.render(…)`,
+ * in which case we avoid that level of indirection for making
stack trace simpler.
+ * But if the exception is another kind, keep it.
+ */
+ if (cause == null || !(e instanceof CannotEvaluateException)) {
+ cause = e;
+ }
+ throw new DataStoreException(e.getLocalizedMessage(), cause);
+ }
+ }
+ return coverage;
+ }
+
+ /**
+ * Whether this resource supports immediate loading of raster data.
+ */
+ private boolean supportImmediateLoading() {
+ return getTileSize().length == TiledGridCoverage.BIDIMENSIONAL;
+ }
+
+ /**
+ * Returns an indication about when the "physical" loading of raster data
will happen.
+ *
+ * @return current raster data loading strategy for this resource.
+ */
+ @Override
+ public final RasterLoadingStrategy getLoadingStrategy() {
+ synchronized (getSynchronizationLock()) {
+ if (loadingStrategy == null) {
+ setLoadingStrategy(supportImmediateLoading());
+ }
+ return loadingStrategy;
+ }
+ }
+
+ /**
+ * Sets the preferred strategy about when to do the "physical" loading of
raster data.
+ *
+ * @param strategy the desired strategy for loading raster data.
+ * @return {@code true} if the given strategy has been accepted, or {@code
false}
+ * if this implementation replaced the given strategy by an
alternative.
+ */
+ @Override
+ public final boolean setLoadingStrategy(final RasterLoadingStrategy
strategy) {
+ synchronized (getSynchronizationLock()) {
+ if (strategy != null) {
+ setLoadingStrategy(strategy ==
RasterLoadingStrategy.AT_READ_TIME && supportImmediateLoading());
+ }
+ return super.setLoadingStrategy(strategy);
+ }
+ }
+
+ /**
+ * Sets the strategy for the given flag.
+ *
+ * @param loadAtReadTime whether all tiles should be read immediately
+ * at {@code read(…)} method call or deferred at a later time.
+ */
+ private void setLoadingStrategy(final boolean loadAtReadTime) {
+ loadingStrategy = loadAtReadTime ? RasterLoadingStrategy.AT_READ_TIME
+ :
RasterLoadingStrategy.AT_RENDER_TIME;
+ }
}
diff --git
a/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
b/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
index b9665e2..105b80e 100644
---
a/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
+++
b/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
@@ -27,6 +27,8 @@ import org.apache.sis.coverage.grid.GridGeometry;
import org.apache.sis.coverage.grid.GridExtent;
import org.apache.sis.storage.DataStoreException;
import org.apache.sis.storage.GridCoverageResource;
+import org.apache.sis.internal.storage.AbstractGridResource;
+import org.apache.sis.internal.storage.RasterLoadingStrategy;
import org.apache.sis.internal.util.StandardDateFormat;
import org.apache.sis.internal.util.Numerics;
import org.apache.sis.image.PixelIterator;
@@ -236,6 +238,16 @@ public strictfp class CoverageReadConsistency extends
TestCase {
}
/**
+ * Applies a random configuration on the resource.
+ */
+ private void randomConfigureResource() {
+ if (resource instanceof AbstractGridResource) {
+ final RasterLoadingStrategy[] choices =
RasterLoadingStrategy.values();
+ ((AbstractGridResource)
resource).setLoadingStrategy(choices[random.nextInt(choices.length)]);
+ }
+ }
+
+ /**
* Creates a random domain to be used as a query on the {@link #resource}
to test.
* All arrays given to this method will have their values overwritten.
*
@@ -291,6 +303,7 @@ public strictfp class CoverageReadConsistency extends
TestCase {
* @throws DataStoreException if an error occurred while using the
resource.
*/
private void readAndCompareRandomRegions() throws DataStoreException {
+ randomConfigureResource();
final GridGeometry gg = resource.getGridGeometry();
final int dimension = gg.getDimension();
final long[] low = new long[dimension];
