Index: Main.java =================================================================== --- Main.java (revision 389) +++ Main.java (working copy) @@ -268,19 +268,8 @@ } else { nameAreas(); } + List areas = areaList.getAreas(); - System.out.println(areas.size() + " areas:"); - for (Area area : areas) { - System.out.format("Area %08d: %d,%d to %d,%d covers %s", - area.getMapId(), - area.getMinLat(), area.getMinLong(), - area.getMaxLat(), area.getMaxLong(), - area.toHexString()); - - if (area.getName() != null) - System.out.print(' ' + area.getName()); - System.out.println(); - } if (kmlOutputFile != null) { File out = new File(kmlOutputFile); @@ -298,6 +287,20 @@ System.err.println("stopped after " + stopAfter); throw new StopNoErrorException("stopped after " + stopAfter); } + + System.out.println(areas.size() + " areas:"); + for (Area area : areas) { + System.out.format("Area %08d: %d,%d to %d,%d covers %s", + area.getMapId(), + area.getMinLat(), area.getMinLong(), + area.getMaxLat(), area.getMaxLong(), + area.toHexString()); + + if (area.getName() != null) + System.out.print(' ' + area.getName()); + System.out.println(); + } + if (keepComplete){ partitionAreasForProblemListGenerator(areas); if ("gen-problem-list".equals(stopAfter)){ @@ -538,7 +541,9 @@ throw new IllegalArgumentException("Error: precomp-sea directory doesn't exist or is not readable: " + precompSeaDir); } } - + if (polygons.isEmpty() == false && numTiles > 0){ + System.out.println("Polygons are ignored because parameter --num-tiles is used"); + } } /** @@ -589,7 +594,7 @@ List areas ; if (numTiles >= 2){ System.out.println("Splitting nodes into " + numTiles + " areas"); - areas = splittableArea.split(polygons, numTiles); + areas = splittableArea.split(numTiles); } else { System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each..."); Index: SplittableDensityArea.java =================================================================== --- SplittableDensityArea.java (revision 389) +++ SplittableDensityArea.java (working copy) @@ -15,13 +15,11 @@ import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap; import it.unimi.dsi.fastutil.ints.IntArrayList; - import java.awt.Point; import java.awt.Rectangle; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; /** @@ -34,39 +32,46 @@ private static final int MAX_LAT_DEGREES = 85; private static final int MAX_LON_DEGREES = 90; public static final int MAX_SINGLE_POLYGON_VERTICES = 40; - private static final int MAX_LOOPS = 100; // number of loops to find better solution private static final int AXIS_HOR = 0; private static final int AXIS_VERT = 1; private static final int NICE_MAX_ASPECT_RATIO = 4; - private static final int MAX_CACHE_SIZE = 200000; + + // higher values mean longer runtime, but also possibly better solutions + private static final int MAX_FAILED_TRIES = 10000000; + + private final DensityMap allDensities; + private final int shift; + private final int maxTileHeight; + private final int maxTileWidth; + + // thrshold values to decide if a solution is good private double maxAspectRatio = Double.MIN_VALUE; private long minNodes = Long.MAX_VALUE; - private final DensityMap allDensities; + private int spread = 0; private double[] aspectRatioFactor; int minAspectRatioFactorPos = Integer.MAX_VALUE; - double maxAspectRatioFactor = Double.MIN_VALUE; - private static final int[] SPREAD_VALUES = { 0, 3, 5, 7}; - private static final int MAX_SPREAD = SPREAD_VALUES[SPREAD_VALUES.length-1]; + private int[] SPREAD_VALUES = { 0, 3, 5, 7 }; + private int maxSpread; private boolean beQuiet = false; private long maxNodes; - private final int shift; - private HashSet cache; - private Int2LongOpenHashMap badMinNodes; + + // helpers to improve performance + private long countBad; + private boolean bruteForce = true; private int [][]yxMap; private int [][]xyMap; - private final int maxTileHeight; - private final int maxTileWidth; + private boolean trimShape; private boolean trimTiles; private boolean allowEmptyPart = false; private int currMapId; - private int maxNodesInDensityMapGridElement; enum sides {TOP,RIGHT,BOTTOM,LEFT} public SplittableDensityArea(DensityMap densities) { this.shift = densities.getShift(); + maxSpread = SPREAD_VALUES[SPREAD_VALUES.length-1]; maxTileHeight = Utils.toMapUnit(MAX_LAT_DEGREES) / (1 << shift); maxTileWidth = Utils.toMapUnit(MAX_LON_DEGREES) / (1 << shift); allDensities = densities; @@ -75,11 +80,10 @@ currMapId = mapId; } - public void setMaxNodes(long maxNodes) { - this.maxNodes = maxNodes; + public void setMaxNodes(long n) { + this.maxNodes = n; } - public void setTrim(boolean trim) { this.trimShape = trim; } @@ -105,7 +109,6 @@ return Collections.emptyList(); prepare(null); Tile startTile = new Tile(0,0,allDensities.getWidth(),allDensities.getHeight(), allDensities.getNodeCount()); - Solution fullSolution = new Solution(spread); Solution startSolution = solveRectangularArea(startTile); @@ -262,8 +265,9 @@ * @param wantedTiles * @return */ - public List split(List namedPolygons, int wantedTiles) { - long currMaxNodes = this.allDensities.getNodeCount() / wantedTiles; + public List split(int wantedTiles) { + long currMaxNodes = this.allDensities.getNodeCount() * 2/ wantedTiles; + class Pair { long maxNodes; int numTiles; @@ -279,10 +283,10 @@ while (true) { setMaxNodes(currMaxNodes); System.out.println("Trying a max-nodes value of " + currMaxNodes + " to split " + allDensities.getNodeCount() + " nodes into " + wantedTiles + " areas"); - List res = split(namedPolygons); + List res = split(); if (res.isEmpty() || res.size() == wantedTiles){ beQuiet = false; - res = split(namedPolygons); + res = split(); return res; } Pair pair = new Pair(currMaxNodes, res.size()); @@ -325,6 +329,8 @@ int minLat = allDensities.getBounds().getMinLat(); int maxLat = allDensities.getBounds().getMaxLat(); int lat = 0; + double maxAspectRatioFactor = Double.MIN_VALUE; + // performance: calculate only once the needed complex math results for (int i = 0; i < aspectRatioFactor.length; i++ ){ lat = minLat + i * (1 << shift); @@ -336,7 +342,7 @@ } } assert lat == maxLat; - maxNodesInDensityMapGridElement = Integer.MIN_VALUE; + int maxNodesInDensityMapGridElement = Integer.MIN_VALUE; int width = allDensities.getWidth(); int height = allDensities.getHeight(); xyMap = new int [width][height]; @@ -564,7 +570,7 @@ * @param tile the tile to be split * @return a solution instance or null */ - private Solution findSolution(int depth, final Tile tile){ + private Solution findSolution(int depth, final Tile tile, long[] parentRowSums, long[] parentColSums, Tile parent){ boolean addAndReturn = false; if (tile.count == 0){ if (!allowEmptyPart) @@ -588,24 +594,53 @@ addAndReturn = true; } else if (tile.width < 2 && tile.height < 2) { return null; - } + } else if (tile.count < 2 * minNodes) + return null; + + if (addAndReturn){ Solution solution = new Solution(spread); solution.add(tile); // can't split further return solution; } - // we have to split the tile IntArrayList offsets = new IntArrayList(); IntArrayList splitXPositions = new IntArrayList(); IntArrayList splitYPositions = new IntArrayList(); - long[] rowSums = null; - long[] colSums = null; - colSums = new long[tile.width]; Arrays.fill(colSums, -1); - rowSums = new long[tile.height];Arrays.fill(rowSums, -1); + + long[] rowSums = new long[tile.height]; + long[] colSums = new long[tile.width]; + if (parent.width == tile.width){ + int srcPos = tile.y - parent.y; + System.arraycopy(parentRowSums, srcPos, rowSums, 0, rowSums.length); + } else + Arrays.fill(rowSums, -1); + if (parent.height == tile.height){ + int srcPos = tile.x - parent.x; + System.arraycopy(parentColSums, srcPos, colSums, 0, colSums.length); + } else + Arrays.fill(colSums, -1); int numTests = 0; int axis = (tile.getAspectRatio() >= 1.0) ? AXIS_HOR:AXIS_VERT; + if (bruteForce){ + int mid = tile.height / 2; + for (int i = 0; i <= mid; i++){ + if (mid + i < tile.height - 1) + splitYPositions.add(mid+i); + if (mid - i > 0 && i != 0) + splitYPositions.add(mid-i); + } + mid = tile.width / 2; + for (int i = 0; i <= mid; i++){ + if (mid + i < tile.width -1) + splitXPositions.add(mid+i); + if (mid - i > 0 && i != 0) + splitXPositions.add(mid-i); + } + numTests = splitXPositions.size() + splitYPositions.size(); + } + else { if (spread == 0){ if (numTests == 0){ offsets.add(0); @@ -612,7 +647,7 @@ numTests = 2; } }else { - if (tile.count > maxNodes * 4 ){ + if (tile.count > maxNodes * 4 && spread > 1){ // jump around int step = tile.width / spread; int pos = step; @@ -629,20 +664,20 @@ numTests = splitXPositions.size() + splitYPositions.size(); } else { offsets.add(0); - for (int i = 1; i < spread; i++){ + for (int i = spread; i >= 1; i--){ offsets.add(i*spread); offsets.add(-i*spread); } - numTests = offsets.size() + 1; - if (spread > 0) - ++numTests; + numTests = offsets.size() * 2 + 1; } } + } + Solution res = null; int currX = 0, currY = 0; - int splitX =-1, splitY = -1; while(numTests-- > 0){ - if (cache.size() > MAX_CACHE_SIZE) + if (countBad > MAX_FAILED_TRIES){ return null; + } Tile[] parts; if (offsets.isEmpty() == false){ @@ -652,26 +687,11 @@ break; } offset = offsets.getInt(currX++); - int pos = splitX + offset; - if (splitX > 0 && (pos >= tile.width || pos <= 0)) - parts = null; - else - parts = tile.splitHorizWithOffset(offset, colSums); - if (parts != null && offset == 0){ - splitX = parts[1].x; - } + parts = tile.splitHorizWithOffset(offset, colSums); } else { offset = offsets.getInt(currY++); - int pos = splitY + offset; - if (splitY > 0 && (pos >= tile.height || pos <= 0)) - parts = null; - else - parts = tile.splitVertWithOffset(offset, rowSums); - if (parts != null && offset == 0){ - splitY = parts[1].y; - } + parts = tile.splitVertWithOffset(offset, rowSums); } - } else { if (axis == AXIS_HOR || currY >= splitYPositions.size()){ if (currX >= splitXPositions.size()) @@ -684,43 +704,53 @@ if (parts == null) continue; Solution part0Sol = null, part1Sol = null; - if (cache.contains(parts[0]) == false && cache.contains(parts[1]) == false){ - if (parts[0].count > parts[1].count){ - // first try the less populated part - Tile help = parts[0]; - parts[0] = parts[1]; - parts[1] = help; - } - if (trimTiles){ - parts[0] = parts[0].trim(); - parts[1] = parts[1].trim(); - } - part0Sol = findSolution(depth + 1, parts[0]); - if (part0Sol == null) - markBad(parts[0]); - else { - part1Sol = findSolution(depth + 1, parts[1]); - if (part1Sol == null) - markBad(parts[1]); - else { - part0Sol.merge(part1Sol); - return part0Sol; - } - } + + if (parts[0].count > parts[1].count){ + // first try the less populated part + Tile help = parts[0]; + parts[0] = parts[1]; + parts[1] = help; } - } - return null; + if (trimTiles){ + parts[0] = parts[0].trim(); + parts[1] = parts[1].trim(); + } + part0Sol = findSolution(depth + 1, parts[0], rowSums, colSums, tile); + if (part0Sol == null){ + markBad(parts[0]); + continue; + } + + part1Sol = findSolution(depth + 1, parts[1], rowSums, colSums, tile); + if (part1Sol == null){ + markBad(parts[1]); + continue; + } + part0Sol.merge(part1Sol); + res = part0Sol; + break; + } + if (parent.width == tile.width){ + int destPos = tile.y - parent.y; + System.arraycopy(rowSums, 0, parentRowSums, destPos, rowSums.length); + } + if (parent.height == tile.height){ + int destPos = tile.x - parent.x; + System.arraycopy(colSums, 0, parentColSums, destPos, colSums.length); + } + return res; } /** - * Store tiles that can't be split into nice parts in the cache. + * Count bad tries * @param tile */ private void markBad(Tile tile){ - cache.add(tile); -// if (cache.size() % 10000 == 0){ + countBad++; + +// if (countBad % 1000000 == 0){ // System.out.println("min-nodes = " + minNodes + ", spread = " -// + spread + ", stored states: " + cache.size()); +// + spread + ", unsuccesful tries: " + countBad); // } } @@ -732,31 +762,44 @@ * @return a solution (maybe be empty) */ private Solution solveRectangularArea(Tile startTile){ - // start values for optimization process (they make sure that we find a solution) + final int MAX_LOOPS = 100; // number of loops to find better solution + + // start values for optimisation process (they make sure that we find a solution) spread = 0; minNodes = 0; - maxAspectRatio = 1L<(); - System.out.println("searching for split with spread " + spread + " and min-nodes " + minNodes); - Solution solution = findSolution(0, startTile); + countBad = 0; + + if (!beQuiet) + System.out.println("searching for split with spread " + spread + " and min-nodes " + minNodes); + Solution solution = findSolution(0, startTile, rowSums, colSums, startTile); + System.out.println("findsol returned after trying " + countBad + " splits"); if (solution != null){ - bestAspectRatio = Math.min(bestAspectRatio, solution.getWorstAspectRatio()); - long bestRating = bestSolution.getRating(); - long rating = solution.getRating(); - if (rating < bestRating || rating == bestRating && solution.getWorstMinNodes() > bestSolution.getWorstMinNodes()) { + if (bestSolution.isEmpty()) { foundBetter = true; - } else if (solution.getWorstMinNodes() > bestSolution.getWorstMinNodes() && solution.isNice()) { + } else if (solution.getWorstMinNodes() > bestSolution.getWorstMinNodes() && (bestSolution.isNice() == false || solution.isNice())) { foundBetter = true; } else if (solution.getWorstAspectRatio() < bestSolution.getWorstAspectRatio() && bestSolution.getWorstMinNodes() < 100){ foundBetter = true; @@ -778,37 +821,34 @@ System.out.println("This can't be improved."); break; } - } - else { + } else { long lastBad = badMinNodes.get(spread); if (lastBad == badMinNodes.defaultReturnValue() || lastBad > minNodes){ badMinNodes.put(spread, minNodes); lastBad = minNodes; } - if ((spread >= MAX_SPREAD && bestSolution.isEmpty() == false)){ + if (((bruteForce || spread >= maxSpread) && bestSolution.isEmpty() == false)){ if (minNodes > bestSolution.getWorstMinNodes() + 1){ + if (bestSolution.spread < maxSpread){ + spread = bestSolution.spread; + lastBad = badMinNodes.get(spread); + } // reduce minNodes minNodes = (bestSolution.getWorstMinNodes() + lastBad) / 2; - if (minNodes - bestSolution.getWorstMinNodes() < 10000) + if (minNodes < bestSolution.getWorstMinNodes() + bestSolution.getWorstMinNodes() / 100){ + // if new minNodes is less than 1% from best, find smallest possible improvement minNodes = bestSolution.getWorstMinNodes() + 1; - if (bestSolution.spread < MAX_SPREAD) - spread = bestSolution.spread; - + maxSpread = SPREAD_VALUES[SPREAD_VALUES.length-1]; + } System.out.println("restarting with min-nodes " + minNodes + " and spread " + spread); continue; } - break; // no hope to find something better in a reasonable time } } - if (foundBetter == false && spread < MAX_SPREAD){ + if (foundBetter == false && spread < maxSpread && bruteForce == false){ // no (better) solution found for the criteria, search also with "non-natural" split lines - for (int i = 0; i < SPREAD_VALUES.length; i++){ - if (spread == SPREAD_VALUES[i]){ - spread = SPREAD_VALUES[i+1]; - break; - } - } + spread = getNextSpread(spread); // System.out.println("Trying non-natural splits with spread " + spread + " ..."); continue; } @@ -817,26 +857,34 @@ // found a correct start, change criteria to find a better(nicer) result maxAspectRatio = Math.max(bestSolution.worstAspectRatio/2, NICE_MAX_ASPECT_RATIO); maxAspectRatio = Math.min(32,maxAspectRatio); - - if (maxAspectRatio == saveMaxAspectRatio){ - if (maxAspectRatio == NICE_MAX_ASPECT_RATIO) { - long lastBad = badMinNodes.get(spread); - if (lastBad == badMinNodes.defaultReturnValue()) - minNodes = bestSolution.getWorstMinNodes() + (maxNodes - bestSolution.getWorstMinNodes() ) / 2; - else + + if (maxAspectRatio == NICE_MAX_ASPECT_RATIO) { + long lastBad = badMinNodes.get(spread); + if (lastBad == badMinNodes.defaultReturnValue()) + minNodes = bestSolution.getWorstMinNodes() + (maxNodes - bestSolution.getWorstMinNodes() ) / 2; + else { + if (lastBad < bestSolution.getWorstMinNodes()){ + badMinNodes.put(spread, minNodes+2); + minNodes = bestSolution.getWorstMinNodes() + 1; + } else minNodes = bestSolution.getWorstMinNodes() + (lastBad - bestSolution.getWorstMinNodes() ) / 2; } - else - minNodes = Math.min(maxNodes / 3, bestSolution.worstMinNodes + maxNodes/20); } + else + minNodes = Math.min(maxNodes / 3, bestSolution.worstMinNodes + maxNodes/20); } + if (bruteForce && numLoops > 5 && (bestSolution.isEmpty() || bestSolution.isNice() == false || bestSolution.getWorstMinNodes() < 0.5 * maxNodes)){ + System.out.println("stopping brute force search"); + bruteForce = false; + badMinNodes = new Int2LongOpenHashMap(); + } + if (saveMaxAspectRatio == maxAspectRatio && saveMinNodes == minNodes && bestSolution.isEmpty() == false && minNodes > bestSolution.getWorstMinNodes() + 1){ + minNodes = bestSolution.getWorstMinNodes() + 1; + } if (saveMaxAspectRatio == maxAspectRatio && saveMinNodes == minNodes){ break; } - if (cache.size() > MAX_CACHE_SIZE){ - break; - } - } + } printFinishMsg(bestSolution); return bestSolution; } @@ -903,8 +951,10 @@ */ public boolean checkSize() { - if (height > maxTileHeight|| width > maxTileWidth) + if (height > maxTileHeight|| width > maxTileWidth){ +// System.out.println("tile is too large: " + this); return false; + } return true; } @@ -981,8 +1031,13 @@ if (splitX < 0){ if (pos == 0 ) splitX = 1; - else - splitX = pos; + else { + // select the split that's closer to target + if (sum - target < target - lastSum && pos + 1 < width) + splitX = pos + 1; + else + splitX = pos; + } } if (offset <= 0 || pos >= splitX + offset) break; @@ -989,6 +1044,10 @@ } } + if (sum > 0 && splitX < 0){ + splitX = width-1; + } + if (splitX + offset <= 0 || splitX + offset >= width) return null; if (offset < 0){ @@ -1049,13 +1108,21 @@ if (splitY < 0){ if (pos == 0 ) splitY = 1; - else - splitY = pos; + else { + // select the split that's closer to target + if (sum - target < target - lastSum && pos + 1 < height) + splitY = pos + 1; + else + splitY = pos; + } } if (offset <= 0 || pos >= splitY + offset) break; } } + if (splitY < 0 && sum > 0){ + splitY = height-1; + } if (splitY + offset <= 0 || splitY + offset >= height) return null; @@ -1215,7 +1282,7 @@ tiles = new ArrayList<>(); this.spread = spread; } - + public boolean add(Tile tile){ tiles.add(tile); double aspectRatio = tile.getAspectRatio(); @@ -1233,35 +1300,14 @@ worstAspectRatio = other.worstAspectRatio; worstMinNodes = other.worstMinNodes; } else { - worstAspectRatio = Math.max(worstAspectRatio, other.worstAspectRatio); - worstMinNodes = Math.min(worstMinNodes, other.worstMinNodes); + if (other.worstAspectRatio > worstAspectRatio) + worstAspectRatio = other.worstAspectRatio; + if (worstMinNodes > other.worstMinNodes) + worstMinNodes = other.worstMinNodes; } tiles.addAll(other.tiles); } - public long getRating(){ - if (tiles.isEmpty()) - return Long.MAX_VALUE; - - int countNonZero = 0; - worstAspectRatio = -1; - worstMinNodes = Long.MAX_VALUE; - for (Tile tile: tiles){ - if (tile.count == 0) - continue; - double aspectRatio = tile.getAspectRatio(); - if (aspectRatio < 1.0) - aspectRatio = 1.0 / aspectRatio; - worstAspectRatio = Math.max(aspectRatio, worstAspectRatio); - worstMinNodes = Math.min(tile.count, worstMinNodes); - ++countNonZero; - } - long rating1 = countNonZero * Math.round(worstAspectRatio*worstAspectRatio); - long rating2 = maxNodes * 100/worstMinNodes; - return rating1 + rating2; - - } - public long getWorstMinNodes(){ return worstMinNodes; } @@ -1288,7 +1334,6 @@ List result = new ArrayList<>(); int minLat = allDensities.getBounds().getMinLat(); int minLon = allDensities.getBounds().getMinLong(); - String note; if (polygonArea != null){ System.out.println("Trying to cut the areas so that they fit into the polygon ..."); @@ -1317,6 +1362,7 @@ } Area area = new Area(r.y,r.x,(int)r.getMaxY(),(int)r.getMaxX()); if (!beQuiet){ + String note; if (tile.count > maxNodes) note = " but is already at the minimum size so can't be split further"; else @@ -1481,17 +1527,22 @@ } public String toString(){ - long rating = getRating(); double ratio = (double) Math.round(worstAspectRatio * 100) / 100; + long percentage = 100 * worstMinNodes / maxNodes; + return tiles.size() + " tile(s). The smallest node count is " + worstMinNodes + " (" + percentage + " %), the worst aspect ratio is near " + ratio; - return tiles.size() + " tile(s) and a rating of " + rating - + ". The smallest node count is " + worstMinNodes + ", the worst aspect ratio is near " + ratio; - } } + private int getNextSpread(int currSpread) { + for (int i = 0; i < SPREAD_VALUES.length; i++){ + if (currSpread == SPREAD_VALUES[i]){ + return SPREAD_VALUES[i+1]; + } + } + return currSpread; + } + } - -