Author: desruisseaux Date: Sat Sep 9 13:43:50 2017 New Revision: 1807904 URL: http://svn.apache.org/viewvc?rev=1807904&view=rev Log: Allow the benchmark to be run from the command line and improve formatting.
Modified: sis/release-test/maven/src/main/java/org/apache/sis/test/referencing/CoordinateOperationComparator.java Modified: sis/release-test/maven/src/main/java/org/apache/sis/test/referencing/CoordinateOperationComparator.java URL: http://svn.apache.org/viewvc/sis/release-test/maven/src/main/java/org/apache/sis/test/referencing/CoordinateOperationComparator.java?rev=1807904&r1=1807903&r2=1807904&view=diff ============================================================================== --- sis/release-test/maven/src/main/java/org/apache/sis/test/referencing/CoordinateOperationComparator.java (original) +++ sis/release-test/maven/src/main/java/org/apache/sis/test/referencing/CoordinateOperationComparator.java Sat Sep 9 13:43:50 2017 @@ -16,9 +16,9 @@ */ package org.apache.sis.test.referencing; -import java.io.FileWriter; +import java.io.Flushable; import java.io.IOException; -import java.io.PrintWriter; +import java.io.PrintStream; import java.util.Random; import org.opengis.geometry.Envelope; import org.opengis.metadata.extent.GeographicBoundingBox; @@ -28,7 +28,6 @@ import org.opengis.referencing.operation import org.opengis.util.FactoryException; import org.apache.sis.geometry.Envelopes; import org.apache.sis.geometry.GeneralEnvelope; -import org.apache.sis.io.wkt.Convention; import org.apache.sis.math.Statistics; import org.apache.sis.metadata.iso.citation.Citations; import org.apache.sis.metadata.iso.extent.DefaultGeographicBoundingBox; @@ -38,13 +37,21 @@ import org.apache.sis.referencing.CRS; import org.apache.sis.referencing.IdentifiedObjects; import org.apache.sis.referencing.crs.AbstractCRS; import org.apache.sis.referencing.cs.AxesConvention; -import org.apache.sis.referencing.operation.transform.AbstractMathTransform; +import org.apache.sis.internal.referencing.Formulas; import org.apache.sis.internal.metadata.ReferencingServices; +import org.apache.sis.io.TableAppender; import org.apache.sis.storage.gdal.Proj4; /** * Compares coordinate operations performed with Apache SIS and {@literal Proj.4}. + * This class is designed for being run from the command line, potentially with output redirected to a file. + * Example: + * + * <blockquote>{@code java org.apache.sis.test.referencing.CoordinateOperationComparator 1 --tabs > result.txt}</blockquote> + * + * The {@code --tabs} option is for using tabulations as column separator instead than formatted outputs. + * This make importation in spreadsheets easier. * * @author Martin Desruisseaux (Geomatys) * @version 0.8 @@ -52,26 +59,31 @@ import org.apache.sis.storage.gdal.Proj4 * @module */ public final class CoordinateOperationComparator { - public static void main(String[] args) throws Exception { - for (int n = 0; ; n++) { - final CoordinateOperationComparator c; - switch (n) { - case 0: c = new CoordinateOperationComparator("Cylindrical Aqual Area (Spherical)", 4053, 3410); break; - case 1: c = new CoordinateOperationComparator("Cylindrical Aqual Area", 4326, 6933); break; - case 2: c = new CoordinateOperationComparator("Pseudo-Mercator", 4326, 3857); break; - case 3: c = new CoordinateOperationComparator("Mercator", 4326, 3395); break; - case 4: c = new CoordinateOperationComparator("Lambert Conic Conformal", 4269, 3978); break; - case 5: c = new CoordinateOperationComparator("Polar stereographic", 4326, 3031); break; - case 6: c = new CoordinateOperationComparator("Albert Equal Area", 4269, 5070); break; - case 7: c = new CoordinateOperationComparator("Mercator 41 to Mercator", 3994, 3395); break; - case 8: c = new CoordinateOperationComparator("Tokyo to JGD2000", 4301, 4612); break; - case 9: c = new CoordinateOperationComparator("Tokyo to JGD2000 in UTM zone 54", 3095, 3100); break; - case 10: c = new CoordinateOperationComparator("OSGB 1936 to ED50 (UKOOA)", 4277, 4230); break; - case 11: c = new CoordinateOperationComparator("Martinique 1938 to RGAF09", 4625, 5489); break; - case 12: c = new CoordinateOperationComparator("Stereographic to stereographic", 2986, 7082); break; - default: return; - } - c.run(); + /** + * Creates one of the predefined tests identified by the given sequential number. All pre-defined tests known + * to the {@link #main(String[]) method} are listed here. This list may be expanded in any future version. + * + * @param n sequential number of the test to create. + * @return the pre-defined test identified by the given number, or {@code null} if none. + */ + private static CoordinateOperationComparator create(final int n) + throws FactoryException, TransformException, IOException + { + switch (n) { + case 1: return new CoordinateOperationComparator("Cylindrical Equal Area (Spherical)", 4053, 3410); + case 2: return new CoordinateOperationComparator("Cylindrical Equal Area", 4326, 6933); + case 3: return new CoordinateOperationComparator("Pseudo-Mercator", 4326, 3857); + case 4: return new CoordinateOperationComparator("Mercator", 4326, 3395); + case 5: return new CoordinateOperationComparator("Lambert Conic Conformal", 4269, 3978); + case 6: return new CoordinateOperationComparator("Polar stereographic", 4326, 3031); + case 7: return new CoordinateOperationComparator("Albert Equal Area", 4269, 5070); + case 8: return new CoordinateOperationComparator("Mercator 41 to Mercator", 3994, 3395); + case 9: return new CoordinateOperationComparator("Tokyo to JGD2000", 4301, 4612); + case 10: return new CoordinateOperationComparator("Tokyo to JGD2000 in UTM zone 54", 3095, 3100); + case 11: return new CoordinateOperationComparator("OSGB 1936 to ED50 (UKOOA)", 4277, 4230); + case 12: return new CoordinateOperationComparator("Martinique 1938 to RGAF09", 4625, 5489); + case 13: return new CoordinateOperationComparator("Stereographic to stereographic", 2986, 7082); + default: return null; } } @@ -100,7 +112,7 @@ public final class CoordinateOperationCo /** * The coordinates to transform as (xâ,yâ), (xâ,yâ), (xâ,yâ), (xâ,yâ), <i>etc.</i> tupples. - * This array will be initialized by {@link #randomCoordinates()}, then never modified. + * This array will be initialized by the constructor, then never modified. */ private final double[] inputs; @@ -114,7 +126,7 @@ public final class CoordinateOperationCo * {@code true} if the CRS is geographic, or {@code false} if the CRS is projected. * If {@code true}, then distances will be estimated using the nautical mile as an approximation. * - * @see #distance(boolean, double, double) + * @see #distance(boolean, double[], double[], int) */ private final boolean isSourceGeographic, isTargetGeographic; @@ -134,27 +146,50 @@ public final class CoordinateOperationCo private final CoordinateOperation[] normalized; /** + * A title for the test to be run, for information purpose only. + */ + private final String title; + + /** + * Names of the implementation being tested, for information purpose only. + */ + private final String[] implementationNames; + + /** * Where to write benchmark results. */ - private final PrintWriter out; + @SuppressWarnings("UseOfSystemOutOrSystemErr") + private static final PrintStream out = System.out; + + /** + * {@code true} for using tabulations instead than formatting in tables. + * This is enabled by the {@code --tabs}} flag on the command-line. + */ + private static boolean useTabulations; + + /** + * {@code true} for flushing the output stream after each line when writing a table. + * This allows more immediate feedback to the user, but sometime break the table layout. + * We disable the immediate mode when writing to a file since the user would not see it anyway. + */ + private static boolean immediate; /** * Creates a comparator for coordinate operations between the given pair of EPSG codes. * Current implementations creates operations for Apache SIS and Proj.4 libraries, * but this list may be expanded in any future version. * - * @param outputFile name of the file where to write results, or {@code null} for standard output. - * If non-null, a {@code ".txt"} extension will be added. + * @param title a title for this benchmark. * @param source EPSG code of source CRS. * @param target EPSG code of target CRS. - * @throws IOException if an error occurred while creating the output file. - * @throws FactoryException if an error occurred while instantiating CRS or coordinate operation. - * @throws TransformException if an error occurred while transforming coordinates. + * @throws FactoryException if an error occurred while instantiating CRS or coordinate operation. + * @throws TransformException if an error occurred while transforming coordinates. + * @throws IOException if an error occurred while writing results. */ - public CoordinateOperationComparator(final String outputFile, final int source, final int target) - throws IOException, FactoryException, TransformException + public CoordinateOperationComparator(final String title, final int source, final int target) + throws FactoryException, TransformException, IOException { - this(outputFile, + this(title, new String[] {"Apache SIS", "Proj.4"}, CRS.findOperation( CRS.forCode( "EPSG:" + source), CRS.forCode( "EPSG:" + target), null), Proj4.createOperation(Proj4.createCRS("+init=epsg:" + source + " +over", 2), @@ -169,14 +204,16 @@ public final class CoordinateOperationCo * All operations except the first (authoritative) one shall use the (<var>longitude</var>, <var>latitude</var>) * axis order, for simpler comparisons with Proj.4 and similar libraries. * - * @param operations the operations to compare. The first operation shall be the authoritative one. - * @throws FactoryException if an error occurred while creating a coordinate operation. - * @throws TransformException if an error occurred while computing the domain of validity. + * @param operations the operations to compare. The first operation shall be the authoritative one. + * @param implementationNames names of the implementation being tested, for information purpose only. + * @throws FactoryException if an error occurred while creating a coordinate operation. + * @throws TransformException if an error occurred while computing the domain of validity. */ - @SuppressWarnings("UseOfSystemOutOrSystemErr") - private CoordinateOperationComparator(final String outputFile, CoordinateOperation... operations) - throws IOException, FactoryException, TransformException + private CoordinateOperationComparator(final String title, final String[] implementationNames, + CoordinateOperation... operations) throws FactoryException, TransformException, IOException { + this.title = title; + this.implementationNames = implementationNames.clone(); operations = operations.clone(); authoritative = operations[0]; final AbstractCRS sourceCRS = AbstractCRS.castOrCopy(authoritative.getSourceCRS()); @@ -194,12 +231,13 @@ public final class CoordinateOperationCo GeographicBoundingBox bbox = CRS.getGeographicBoundingBox(authoritative); bbox = Extents.intersection(bbox, new DefaultGeographicBoundingBox(-179, 179, -89, 89)); final Envelope domain = Envelopes.transform(new GeneralEnvelope(bbox), normalizedSource); - out = (outputFile != null) ? new PrintWriter(new FileWriter(outputFile + ".txt")) : new PrintWriter(System.out); - print("Source CRS", sourceCRS); - print("Target CRS", targetCRS); - print("Operation", authoritative); - print("Method", method(authoritative)); - out.println("Domain:\t" + domain); + final Appendable table = newTable(); + printIdentification(table, "Source CRS", sourceCRS); + printIdentification(table, "Target CRS", targetCRS); + printIdentification(table, "Operation", authoritative); + printIdentification(table, "Method", method(authoritative)); + table.append(String.format("Domain:\t\t%s%n", domain)); + flush(table); /* * Fills the input array with random coordinates. * This array shall not be modified after construction. @@ -231,16 +269,6 @@ public final class CoordinateOperationCo } /** - * Prints the name of the given identified objects. - * - * @param label label to write before the identified object. - * @param object object for which to write the name. - */ - private void print(final String label, final IdentifiedObject object) { - out.printf("%s:\t%s\t%s%n", label, IdentifiedObjects.toString(IdentifiedObjects.getIdentifier(object, Citations.EPSG)), object.getName().getCode()); - } - - /** * Get the operation method of the given coordinate operation. * This is used for information purpose only. */ @@ -258,17 +286,42 @@ public final class CoordinateOperationCo } /** + * Compares the coordinate operation results of Apache SIS with other implementations. + * This comparisons is performed before to execute the actual benchmark, so it already + * causes some JVM warmup. For all comparisons, Apache SIS is taked as the reference. + */ + private void compareOperationResults() throws TransformException, IOException { + out.printf("Difference in forward operation results between %s and other implementations:%n", implementationNames[0]); + final Statistics stats = new Statistics("Difference"); + final Appendable table = newTable(); + table.append(String.format("Implementation\tDifference (m)\tStd. dev.%n")); + final int numPts = inputs.length / DIM; + normalized[0].getMathTransform().transform(inputs, 0, outputs, 0, numPts); + final double[] intermediate = new double[inputs.length]; + for (int j=1; j<normalized.length; j++) { + normalized[j].getMathTransform().transform(inputs, 0, intermediate, 0, numPts); + for (int i = intermediate.length; (i -= DIM) >= 0;) { + stats.accept(distance(isTargetGeographic, outputs, intermediate, i)); + } + table.append(String.format("%s\t%g\t%g%n", implementationNames[j], stats.mean(), stats.standardDeviation(false))); + stats.reset(); + } + flush(table); + } + + /** * Runs the benchmarks. First, this method compare operation results without measuring performance. * Then, this method performs "forward operation" followed by "inverse operation" one hundred times, * measuring performances and drifts at each iteration. * * @throws TransformException if an error occurred while transforming coordinates. - * @throws IOException if an error occurred while writing results. + * @throws IOException if an error occurred while writing results. */ public void run() throws TransformException, IOException { compareOperationResults(); - for (final CoordinateOperation op : normalized) { - measure(op); + for (int j=0; j<normalized.length; j++) { + out.printf("Testing %s with %s implementation%n", title, implementationNames[j]); + measure(normalized[j]); } if (out.checkError()) { throw new IOException("Error while writing results file."); @@ -276,55 +329,24 @@ public final class CoordinateOperationCo } /** - * Compares the coordinate operation results of Apache SIS with other implementations. - */ - private void compareOperationResults() throws TransformException { - final int numPts = inputs.length / DIM; - normalized[0].getMathTransform().transform(inputs, 0, outputs, 0, numPts); - final Statistics[] stats = new Statistics[normalized.length - 1]; - for (int i=0; i<stats.length; i++) { - stats[i] = new Statistics("Difference"); - } - out.println(); - final double[] intermediate = new double[inputs.length]; - for (int j=0; j<stats.length; j++) { - final Statistics s = stats[j]; - normalized[j+1].getMathTransform().transform(inputs, 0, intermediate, 0, numPts); - for (int i = intermediate.length; (i -= DIM) >= 0;) { - s.accept(distance(isTargetGeographic, outputs, intermediate, i)); - } - } - out.print("Difference in projection results (metres):"); - for (final Statistics s : stats) { - out.printf("\t%g\t±%g", s.mean(), s.standardDeviation(false)); - } - out.println(); - } - - /** * Measures performance and drift of the given coordinate operation. */ - private void measure(final CoordinateOperation op) throws TransformException { + private void measure(final CoordinateOperation op) throws TransformException, IOException { System.arraycopy(inputs, 0, outputs, 0, inputs.length); final int numPts = inputs.length / DIM; final MathTransform tr = op.getMathTransform(); final MathTransform inverse = tr.inverse(); - final Statistics drift = new Statistics("Drift"); + final Statistics drift = new Statistics("Drift (m)"); final Statistics forwardTime = new Statistics("Forward time (ms)"); final Statistics inverseTime = new Statistics("Inverse time (ms)"); final Statistics cumulatedTime = new Statistics("Cumulated time (ms)"); - out.println(); - out.println("ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ"); - out.println("Measuring drift after " + NUM_LOOPS + " iterations for:"); + out.println("Implementation defines the MathTransform as below:"); out.println(); out.println(tr); - if (tr instanceof AbstractMathTransform) { - out.println(); - out.println("Internal:"); - out.println(((AbstractMathTransform) tr).toString(Convention.INTERNAL)); - } out.println(); - out.println("Mean\tStd. dev.\tMinimum\tMaximum\tForward time (ms)\tInverse time (ms)"); + out.println("Drift after up to " + NUM_LOOPS + " iterations:"); + Appendable table = newTable(); + table.append(String.format("Mean (m)\tStd. dev.\tMinimum (m)\tMaximum (m)\tForward time (ms)\tInverse time (ms)%n")); for (int n=0; n<NUM_LOOPS; n++) { final long t0 = System.nanoTime(); tr.transform(outputs, 0, outputs, 0, numPts); @@ -335,7 +357,10 @@ public final class CoordinateOperationCo final double tf = (t1 - t0) / (double) StandardDateFormat.NANOS_PER_MILLISECOND; final double ti = (t2 - t1) / (double) StandardDateFormat.NANOS_PER_MILLISECOND; final double tc = (t2 - t0) / (double) StandardDateFormat.NANOS_PER_MILLISECOND; - out.printf("%g\t%g\t%g\t%g\t%g\t%g%n", drift.mean(), drift.standardDeviation(false), drift.minimum(), drift.maximum(), tf, ti); + + if (n != 0 && immediate) ((Flushable) table).flush(); + table.append(String.format("%g\t%g\t%g\t%g\t%g\t%g%n", + drift.mean(), drift.standardDeviation(false), drift.minimum(), drift.maximum(), tf, ti)); if (n >= WARMUP_LOOPS) { forwardTime .accept(tf); inverseTime .accept(ti); @@ -343,12 +368,23 @@ public final class CoordinateOperationCo } drift.reset(); } - out.printf("Average execution time (forward):\t%g\t± %g%n", forwardTime.mean(), inverseTime.standardDeviation(false)); - out.printf("Average execution time (inverse):\t%g\t± %g%n", inverseTime.mean(), inverseTime.standardDeviation(false)); - out.printf("Average execution time (cumulated):\t%g\t± %g%n", cumulatedTime.mean(), cumulatedTime.standardDeviation(false)); - out.println(); - out.println("Performance (ms):"); - out.println("Block size\tForward\tStd. dev.\tInverse\tStd. dev.\tCumulated\tStd. dev."); + flush(table); + out.println("Average execution time:"); + table = newTable(); + table.append(String.format("\tMean (ms)\tStd. dev%n")); + table.append(String.format("Forward:\t%g\t%g%n", forwardTime.mean(), forwardTime.standardDeviation(false))); + table.append(String.format("Inverse:\t%g\t%g%n", inverseTime.mean(), inverseTime.standardDeviation(false))); + table.append(String.format("Cumulated:\t%g\t%g%n", cumulatedTime.mean(), cumulatedTime.standardDeviation(false))); + flush(table); + /* + * Test with decreasing amount of coordinates transformed in one method call. + * This increase the cost of determining which operation to execute. + * We measure performance degradation caused by this increasing cost. + */ + out.println("Performance with decreasing amount of coordinates processed in one method call:"); + table = newTable(); + table.append(String.format("Block size\tForward time (ms)\tStd. dev.\tInverse time (ms)\tStd. dev.\tCumulated (ms)\tStd. dev.%n")); + boolean first = true; for (int size = inputs.length / DIM; size >= 1; size /= 2) { forwardTime.reset(); inverseTime.reset(); @@ -370,13 +406,19 @@ public final class CoordinateOperationCo inverseTime .accept(ti / (double) StandardDateFormat.NANOS_PER_MILLISECOND); cumulatedTime.accept(tc / (double) StandardDateFormat.NANOS_PER_MILLISECOND); } - out.printf("%d\t%g\t%g\t%g\t%g\t%g\t%g\n", size, + if (!first && immediate) ((Flushable) table).flush(); + table.append(String.format("%d\t%g\t%g\t%g\t%g\t%g\t%g\n", size, forwardTime.mean(), forwardTime.standardDeviation(false), inverseTime.mean(), inverseTime.standardDeviation(false), - cumulatedTime.mean(), cumulatedTime.standardDeviation(false)); + cumulatedTime.mean(), cumulatedTime.standardDeviation(false))); collectDifferences(drift); + first = false; + } + flush(table); + if (drift.maximum() > Formulas.LINEAR_TOLERANCE) { + out.println("WARNING: large drift in above performance check"); + out.println(drift); } - out.printf("Verification: average difference = %g and maximal difference = %g%n", drift.mean(), drift.maximum()); } /** @@ -409,4 +451,117 @@ public final class CoordinateOperationCo } return d; } + + /** + * Returns the destination where to write next tabular data. + */ + private static Appendable newTable() { + if (useTabulations) return out; + TableAppender table = new TableAppender(out); + table.appendHorizontalSeparator(); + return table; + } + + /** + * Flushes the output created by {@link #newTable()} and append a new line. + * After this method call, the table should not be used anymore. + */ + private static void flush(final Appendable table) throws IOException { + if (table instanceof TableAppender) { + ((TableAppender) table).appendHorizontalSeparator(); + } + ((Flushable) table).flush(); + out.println(); + } + + /** + * Prints the name of the given identified objects. + * + * @param label label to write before the identified object. + * @param object object for which to write the name. + */ + private static void printIdentification(final Appendable table, final String label, final IdentifiedObject object) throws IOException { + String id = IdentifiedObjects.toString(IdentifiedObjects.getIdentifier(object, Citations.EPSG)); + table.append(String.format("%s:\t%s\t%s%n", label, (id != null) ? id : "", object.getName().getCode())); + } + + /** + * Runs the benchmark identified by the given number. Each number identifies a particular pair of source + * and target CRS. For example the test #1 applies the "Cylindrical Equal Area (Spherical)" projection. + * See source code of this class for the list of available tests. + * + * @param args number of the test to execute. + * @throws FactoryException if an error occurred while instantiating CRS or coordinate operation. + * @throws TransformException if an error occurred while transforming coordinates. + * @throws IOException if an error occurred while writing results. + */ + public static void main(final String[] args) throws FactoryException, TransformException, IOException { + /* + * Where to write error messages or information. While this stream is called "the error stream", it is actually + * also used for printing information (for example progress) that we don't want to include in the result file. + * Note that this is also the stream used by {@code java.util.logging}. + */ + @SuppressWarnings("UseOfSystemOutOrSystemErr") + final PrintStream info = System.err; + immediate = (System.console() != null); + + String error = null; + int sourceCRS = 0, targetCRS = 0; + for (final String p : args) { + if (p.equals("--tabs")) { + useTabulations = true; + } else if (p.startsWith("--")) { + error = String.format("Unrecognized option: %s", p); + break; + } else { + final int n; + try { + n = Integer.parseInt(args[0]); + } catch (NumberFormatException e) { + error = String.format("Invalid test number or EPSG code: %s", e.getLocalizedMessage()); + break; + } + if (n <= 0) { + error = String.format("Invalid test number or EPSG code: %d", n); + break; + } + if (sourceCRS == 0) { + sourceCRS = n; + } else if (targetCRS == 0) { + targetCRS = n; + } else { + error = "Too many EPSG codes (expected 2)."; + break; + } + } + } + /* + * If there is two numbers, they are interpreted as EPSG codes of source and target CRS respectively. + * But if there is only one number (targetCRS == 0), then the source CRS is interpreted as the number + * of a pre-defined test. + */ + CoordinateOperationComparator c = null; + if (error == null) { + if (sourceCRS == 0) { + error = String.format("Usage: one or two numbers%n" + + " [sequential number of pre-defined test]%n" + + " [EPSG code of source CRS] [EPSG code of targetCRS]"); + } else if (targetCRS == 0) { + info.printf("Initializing pre-defined test #%d...%n", sourceCRS); + c = create(sourceCRS); + if (c == null) { + error = "Error: no such pre-defined test."; + } + } else { + String name = String.format("EPSG:%d to EPSG:%d", sourceCRS, targetCRS); + info.printf("Initializing test for %s...%n", name); + c = new CoordinateOperationComparator(name, sourceCRS, targetCRS); + } + } + if (c == null) { + info.println(error); + System.exit(1); + } + c.run(); + } }