hi list,
her now my newst version from patch
first a smal benchmark with local data files (no download)
rendering from a city cell
Fork=2 vs. Cores=4 on a quadcore system
workspace on harddisk
Fork=2
real 14m4.269s
user 17m9.113s
sys 1m28.356s
Cores=4
real 5m12.736s
user 12m14.226s
sys 1m29.026s
i think this say all ;)
ok now all new features:
* threaded optimizePNG
* new threaded Renderer
* new test that kill threads befor he reder to large data files
a 16mb data.osm file consum ~ 1gb ram per renderer so i lets him not start 4
renderer on a 2gb ram machine this save the renderer a bit (its not the best
way)
new config options:
Cores=x
0 or Fork>0 disable threading
MaxMemory=3000
in k 3000 work on my 4gb system perfectly
to the developer: please test it! ;)
i have it testet in xy mod and in loop mod
i hove i have all timing prolemas, killed resorce problems and the GD lib
problems fixed ;)
to the testers! search useTestDirData in lib/Tileset.pm
bugs:
the warning "Scalars leaked: 1"
http://sunsite.ualberta.ca/Documentation/Misc/perl-5.6.1/pod/perldiag.html#item_Scalars_leaked:_%d
i tink he men $self, all childs have a copy and not one can freeing his
René
diff -urbBdpN a/lib/optimizePngTasks.pm b/lib/optimizePngTasks.pm
--- a/lib/optimizePngTasks.pm 1970-01-01 01:00:00.000000000 +0100
+++ b/lib/optimizePngTasks.pm 2008-11-12 19:24:50.000000000 +0100
@@ -0,0 +1,261 @@
+package optimizePngTasks;
+
+
+use warnings;
+use strict;
+use File::Temp qw/ tempfile tempdir /;
+use File::Copy;
+use File::Path;
+use Error qw(:try);
+use TahConf;
+use threads;
+use Thread::Semaphore;
+
+
+sub new
+{
+ my $class = shift;
+ my $Config = TahConf->getConfig();
+
+ my %sharedStack :shared;
+ my @sharedFilelist :shared;
+ my @sharedTranslist :shared;
+ my $self = {
+ Config => $Config,
+ SHARED => \%sharedStack,
+ DESTROYED => 0,
+ children => [],
+ };
+
+ $self->{SHARED}->{DESTROYED} = 0;
+ $self->{SHARED}->{JOBS} = -1;
+ $self->{SHARED}->{JOBSREADY} = 0;
+ $self->{SHARED}->{JOBSFILES} = [EMAIL PROTECTED];
+ $self->{SHARED}->{JOBSTRANSPARENT} = [EMAIL PROTECTED];
+ $self->{SHARED}->{CHILDCRASH} = 0; # TODO:
+
+ bless $self, $class;
+ return $self;
+}
+
+
+sub DESTROY
+{
+
+}
+
+########
+# kill all children threads
+########
+sub killAllChilds
+{
+ my $self = shift;
+ my $Config = $self->{Config};
+
+ # set the destroy flag (detached childrens!)
+ $self->{'Semaphore'}->down();
+ $self->{SHARED}->{DESTROYED} = 1;
+ $self->{'Semaphore'}->up();
+
+}
+
+
+##########
+# start and init my children
+##########
+sub startChildren {
+ my $self = shift;
+ my $Config = $self->{Config};
+
+ if ($Config->get("Cores")) {
+ $self->{'maxChildren'} = $Config->get("Cores");
+ $self->{'children'} = [];
+ $self->{'Semaphore'} = Thread::Semaphore->new();
+
+ $::currentSubTask ='optimize';
+ $::progressPercent = 0;
+ ::statusMessage("init ". $self->{'maxChildren'} ." optimizePNG Child Tasks", 0, 6);
+ $self->{SHARED}->{'progress'} = 0;
+ for my $childID (1 .. $self->{'maxChildren'}) {
+ $self->{'children'}->[$childID] = threads->create( sub {
+ threads->detach();
+ my $sleeping = 0;
+ while (!$self->{SHARED}->{DESTROYED}) {
+
+
+ sleep 1;
+
+ while ($self->{SHARED}->{JOBS} < $#{$self->{SHARED}->{JOBSFILES}}) {
+ my ($png_file, $transparent) = "";
+
+ # access: lock()
+ $self->{'Semaphore'}->down();
+ $self->{SHARED}->{JOBS}++;
+
+ $self->{SHARED}->{'progress'}++;
+ if($#{$self->{SHARED}->{JOBSFILES}} >0) {
+ $::progressPercent = 100 * $self->{SHARED}->{'progress'} / ($#{$self->{SHARED}->{JOBSFILES}}+1);
+ }
+
+ $png_file = $self->{SHARED}->{JOBSFILES}->[ $self->{SHARED}->{JOBS} ];
+ $transparent = $self->{SHARED}->{JOBSTRANSPARENT}->[ $self->{SHARED}->{JOBS} ];
+
+
+ # access: unlock()
+ $self->{'Semaphore'}->up();
+
+ if( $png_file) { # no new work? go sleeping
+ #####
+ # lets do my work now
+ #####
+
+ $self->optimizePngClient($png_file, $transparent);
+
+ $self->{'Semaphore'}->down();
+ $self->{SHARED}->{JOBSREADY}++;
+ $self->{'Semaphore'}->up();
+ }
+ }
+
+
+ }
+ ::statusMessage("optimizePNG child $childID exit" ,1,10);
+
+ }
+ ); # threads->create(sub) end
+ } # for
+
+ }
+
+} # sub startChildren
+
+# add a new job ::addJob->($png_file,$transparent)
+sub addJob {
+ my $self = shift;
+ my $Config = $self->{Config};
+ my $png_file = shift;
+ my $transparent = shift;
+
+ $self->{'Semaphore'}->down();
+
+ my $pos = $#{$self->{SHARED}->{JOBSFILES}}+1;
+
+ $self->{SHARED}->{JOBSFILES}->[ $pos ] = $png_file;
+ $self->{SHARED}->{JOBSTRANSPARENT}->[ $pos ] = $transparent;
+
+ $self->{'Semaphore'}->up();
+}
+
+
+# wait of all my jobs
+sub wait {
+ my $self = shift;
+
+ ::statusMessage("Wait of my PNG optimize Children", 0, 6);
+
+ while ($self->{SHARED}->{JOBSREADY} <= $#{$self->{SHARED}->{JOBSFILES}}) {
+ sleep 1;
+# print $self->{SHARED}->{JOBSREADY} ." ".$#{$self->{SHARED}->{JOBSFILES}} ."\n";
+ }
+
+}
+
+# reset my lists
+sub dataReset {
+ my $self = shift;
+
+ $self->{'Semaphore'}->down();
+
+ $self->{SHARED}->{JOBS} = -1;
+ $self->{SHARED}->{JOBSREADY} = 0;
+ undef @{ $self->{SHARED}->{JOBSFILES} };
+ undef @{ $self->{SHARED}->{JOBSTRANSPARENT} };
+
+ $self->{SHARED}->{'progress'} = 0;
+
+ $self->{'Semaphore'}->up();
+}
+
+
+#-----------------------------------------------------------------------------
+# optimize a PNG file
+#
+# Parameters:
+# $png_file - file name of PNG file
+# $transparent - whether or not this is a transparent tile
+#-----------------------------------------------------------------------------
+sub optimizePngClient
+{
+ my $self = shift();
+ my $png_file = shift();
+ my $transparent = shift();
+
+ my $Config = $self->{Config};
+
+ my $optipngOptions = "-l 9";
+
+ my $redirect = ($^O eq "MSWin32") ? "" : ">/dev/null";
+ my $tmp_suffix = '.cut';
+ my $tmp_file = $png_file . $tmp_suffix;
+ my (undef, undef, $png_file_name) = File::Spec->splitpath($png_file);
+
+ my $cmd;
+ if ($transparent) {
+ # Don't quantize if it's transparent
+ rename($png_file, $tmp_file);
+ }
+ elsif (($Config->get("PngQuantizer")||'') eq "pngnq") {
+ $cmd = sprintf("\"%s\" -e .png%s -s1 -n256 %s %s",
+ $Config->get("pngnq"),
+ $tmp_suffix,
+ $png_file,
+ $redirect);
+
+ ::statusMessage("ColorQuantizing $png_file_name", 0, 6);
+ if(::runCommand($cmd, $::PID)) {
+ # Color quantizing successful
+ unlink($png_file);
+ }
+ else {
+ # Color quantizing failed
+ ::statusMessage("ColorQuantizing $png_file_name with ".$Config->get("PngQuantizer")." failed", 1, 0);
+ rename($png_file, $tmp_file);
+ }
+ }
+ else {
+ ::statusMessage("Not Color Quantizing $png_file_name, pngnq not installed?", 0, 6);
+ rename($png_file, $tmp_file);
+ }
+
+ if ($Config->get("PngOptimizer") eq "pngcrush") {
+ $cmd = sprintf("\"%s\" %s -q %s %s %s",
+ $Config->get("Pngcrush"),
+ $optipngOptions,
+ $tmp_file,
+ $png_file,
+ $redirect);
+ }
+ elsif ($Config->get("PngOptimizer") eq "optipng") {
+ $cmd = sprintf("\"%s\" %s -out %s %s", #no quiet, because it even suppresses error output
+ $Config->get("Optipng"),
+ $tmp_file,
+ $png_file,
+ $redirect);
+ }
+ else {
+ ::statusMessage("PngOptimizer not configured (should not happen, update from svn, and check config file)", 1, 0);
+ ::talkInSleep("Install a PNG optimizer and configure it.", 15);
+ }
+
+ ::statusMessage("Optimizing $png_file_name", 0, 6);
+ if(::runCommand($cmd, $::PID)) {
+ unlink($tmp_file);
+ }
+ else {
+ ::statusMessage("Optimizing $png_file_name with " . $Config->get("PngOptimizer") . " failed", 1, 0);
+ rename($tmp_file, $png_file);
+ }
+}
+
+
+1;
diff -urbBdpN a/lib/Tileset.pm b/lib/Tileset.pm
--- a/lib/Tileset.pm 2008-11-12 18:58:51.573154294 +0100
+++ b/lib/Tileset.pm 2008-11-12 20:33:13.000000000 +0100
@@ -30,6 +30,11 @@ use File::Copy;
use File::Path;
use GD 2 qw(:DEFAULT :cmp);
+use threads;
+use Thread::Semaphore;
+use optimizePngTasks;
+
+
#-----------------------------------------------------------------------------
# creates a new Tileset instance and returns it
# parameter is a request object with x,y,z, and layer atributes set
@@ -62,24 +67,7 @@ sub new
# create true color images by default
GD::Image->trueColor(1);
- # create blank comparison images
- my $EmptyLandImage = new GD::Image(256,256);
- my $MapLandBackground = $EmptyLandImage->colorAllocate(248,248,248);
- $EmptyLandImage->fill(127,127,$MapLandBackground);
-
- my $EmptySeaImage = new GD::Image(256,256);
- my $MapSeaBackground = $EmptySeaImage->colorAllocate(181,214,241);
- $EmptySeaImage->fill(127,127,$MapSeaBackground);
-
- # Some broken versions of Inkscape occasionally produce totally black
- # output. We detect this case and throw an error when that happens.
- my $BlackTileImage = new GD::Image(256,256);
- my $BlackTileBackground = $BlackTileImage->colorAllocate(0,0,0);
- $BlackTileImage->fill(127,127,$BlackTileBackground);
- $self->{EmptyLandImage} = $EmptyLandImage;
- $self->{EmptySeaImage} = $EmptySeaImage;
- $self->{BlackTileImage} = $BlackTileImage;
# Inkscape auto-backup/reset setup
# Takes a backup copy of ~/.inkscape/preferences.xml if
@@ -119,12 +107,8 @@ sub new
#-----------------------------------------------------------------------------
sub DESTROY
{
- my $self = shift;
- # Don't clean up in child threads
- return if ($self->{childThread});
+# perl call DESTROY more as on time!
- # only cleanup if we are the parent thread
- $self->cleanup();
}
#-----------------------------------------------------------------------------
@@ -143,6 +127,141 @@ sub generate
$self->{bbox}= bbox->new(ProjectXY($req->ZXY));
+
+
+ #########
+ # init Children for optimizePngTasks and threadedRenderer
+ #########
+ if ($Config->get("Cores") && !$Config->get("Fork") ) {
+
+ # start optimizePng Childs
+ $self->{optimizePngTasks} = optimizePngTasks->new();
+ $self->{optimizePngTasks}->startChildren();
+
+
+ # add renderer childs
+ $self->{'maxChildren'} = $Config->get("Cores");
+ $self->{'rendererChildren'} = [];
+ $self->{'rendererSemaphore'} = Thread::Semaphore->new();
+
+ my %sharedStack :shared;
+ my @sharedJobs :shared;
+ my @sharedJobsLayer :shared;
+ my @sharedJobsLayerDataFile :shared;
+ my @childStop :shared;
+
+ $self->{SHARED} = \%sharedStack;
+ $self->{SHARED}->{DESTROYED} = 0;
+ $self->{SHARED}->{RENDERERJOBS} = [EMAIL PROTECTED];
+ $self->{SHARED}->{RENDERERJOBLAYER} = [EMAIL PROTECTED];
+ $self->{SHARED}->{RENDERERJOBLAYERDATA} = [EMAIL PROTECTED];
+ $self->{SHARED}->{RENDERERJOBSPOS} = -1;
+ $self->{SHARED}->{RENDERERJOBSREADY} = 0;
+ $self->{SHARED}->{CHILDSTOP} = [EMAIL PROTECTED]; # for stop single clients
+
+ ::statusMessage("init ". $self->{'maxChildren'} ." Renderer Child Tasks", 0, 6);
+
+ for my $childID (1 .. $self->{'maxChildren'}) {
+ $self->{SHARED}->{CHILDSTOP}->[$childID] = 0;
+ $self->{'rendererChildren'}->[$childID] = threads->create( sub {
+
+ threads->detach();
+
+ my $req = $self->{req};
+ my $Config = $self->{Config};
+ my $pos;
+ my $layer;
+ my $layerDataFile;
+ my $zoom;
+
+ # create blank comparison images
+ my $EmptyLandImage = new GD::Image(256,256);
+ my $MapLandBackground = $EmptyLandImage->colorAllocate(248,248,248);
+ $EmptyLandImage->fill(127,127,$MapLandBackground);
+
+ my $EmptySeaImage = new GD::Image(256,256);
+ my $MapSeaBackground = $EmptySeaImage->colorAllocate(181,214,241);
+ $EmptySeaImage->fill(127,127,$MapSeaBackground);
+
+ # Some broken versions of Inkscape occasionally produce totally black
+ # output. We detect this case and throw an error when that happens.
+ my $BlackTileImage = new GD::Image(256,256);
+ my $BlackTileBackground = $BlackTileImage->colorAllocate(0,0,0);
+ $BlackTileImage->fill(127,127,$BlackTileBackground);
+
+ $self->{EmptyLandImage} = $EmptyLandImage;
+ $self->{EmptySeaImage} = $EmptySeaImage;
+ $self->{BlackTileImage} = $BlackTileImage;
+
+ # wait of the global destroy flag or of the singel stop flag
+ while(!$self->{SHARED}->{DESTROYED} && !$self->{SHARED}->{CHILDSTOP}->[$childID] ) {
+
+ while ($self->{SHARED}->{RENDERERJOBSPOS} < $#{ $self->{SHARED}->{RENDERERJOBS} }) {
+
+ # access: lock()
+ $self->{'rendererSemaphore'}->down();
+
+ $self->{SHARED}->{RENDERERJOBSPOS}++;
+ $pos = $self->{SHARED}->{RENDERERJOBSPOS};
+ $zoom = $self->{SHARED}->{RENDERERJOBS}->[$pos];
+ $layer = $self->{SHARED}->{RENDERERJOBLAYER}->[$pos];
+ $layerDataFile = $self->{SHARED}->{RENDERERJOBLAYERDATA}->[$pos];
+
+ # access: unlock()
+ $self->{'rendererSemaphore'}->up();
+
+ ::statusMessage("Rendererclient $childID get job $pos zoom $zoom on layer $layer $layerDataFile" ,1,10);
+
+ ####
+ # i do my work now
+ ####
+ $self->Render($layer, $zoom, $layerDataFile);
+
+ $self->{'rendererSemaphore'}->down();
+ $self->{SHARED}->{RENDERERJOBSREADY}++;
+ $self->{'rendererSemaphore'}->up();
+ }
+
+ sleep 1;
+ }
+ ::statusMessage("Renderer child $childID exit" ,1,10);
+ } #sub;
+ ); #threads->create
+ } # for
+
+ }
+ else {
+ $self->{optimizePngTasks} = undef;
+ $self->{'rendererChildren'} = undef;
+ }
+
+
+
+
+ # create blank comparison images
+ my $EmptyLandImage = new GD::Image(256,256);
+ my $MapLandBackground = $EmptyLandImage->colorAllocate(248,248,248);
+ $EmptyLandImage->fill(127,127,$MapLandBackground);
+
+ my $EmptySeaImage = new GD::Image(256,256);
+ my $MapSeaBackground = $EmptySeaImage->colorAllocate(181,214,241);
+ $EmptySeaImage->fill(127,127,$MapSeaBackground);
+
+ # Some broken versions of Inkscape occasionally produce totally black
+ # output. We detect this case and throw an error when that happens.
+ my $BlackTileImage = new GD::Image(256,256);
+ my $BlackTileBackground = $BlackTileImage->colorAllocate(0,0,0);
+ $BlackTileImage->fill(127,127,$BlackTileBackground);
+
+ $self->{EmptyLandImage} = $EmptyLandImage;
+ $self->{EmptySeaImage} = $EmptySeaImage;
+ $self->{BlackTileImage} = $BlackTileImage;
+
+
+
+
+
+
::statusMessage(sprintf("Tileset (%d,%d,%d) around %.2f,%.2f", $req->ZXY, $self->{bbox}->center), 1, 0);
if($req->Z >= 12)
@@ -152,9 +271,43 @@ sub generate
#------------------------------------------------------
my $beforeDownload = time();
- my $FullDataFile = $self->downloadData($req->layers);
+
+ # TODO: FIXME: remove it on the stable version! only for debuging and tests else { its the original}
+ # add a optional testadata directory (download not the data)
+ # DO NOT UPLOAD THE RESULTS REAL!
+ my $testdatadir = File::Spec->join($Config->get("WorkingDirectory"), 'testdatadir');
+ my $testdatafile = File::Spec->join($testdatadir, 'data.osm');
+ my $FullDataFile = "";
+
+ if($Config->get("useTestDirData") && $Config->get("debug") && $Config->get("UploadToDirectory")
+ && -d $testdatadir && -f $testdatafile)
+ {
+ $FullDataFile = File::Spec->join($self->{JobDir}, 'data.osm');
+ copy($testdatafile,$FullDataFile)
+ }
+ else {
+ $FullDataFile = $self->downloadData($req->layers);
+ }
+
::statusMessage("Download in ".(time() - $beforeDownload)." sec",1,10);
+ # manage renderer memory usage
+ # a 16mb osm file consum ca 1gb ram as svg "int(16786037/1024/16)"
+ my @datafileStats = stat( $FullDataFile );
+ my $caMemoryUsage = $datafileStats[7]/1024/16;
+
+ if( ( $caMemoryUsage*$self->{'maxChildren'} ) > $Config->get("MaxMemory")) {
+ # too little memory
+ my $newMaxChildren = int($Config->get("MaxMemory")/$caMemoryUsage);
+ $newMaxChildren = 1 if $self->{'maxChildren'} eq $newMaxChildren;
+
+ ::statusMessage("too little memory for the render job and ". $self->{'maxChildren'} ." Children, stopp ". ($self->{'maxChildren'}-$newMaxChildren) ." renderer childs from ". $self->{'maxChildren'} ,1,10);
+
+ for(my $stopCount = ($self->{'maxChildren'}-$newMaxChildren); $stopCount>0;$stopCount-- ) {
+ $self->{SHARED}->{CHILDSTOP}->[$stopCount]=1;
+ }
+ }
+
#------------------------------------------------------
# Handle all layers, one after the other
#------------------------------------------------------
@@ -321,7 +474,23 @@ sub generate
$::currentSubTask = "";
::keepLog($$,"GenerateTileset","stop",'x='.$req->X.',y='.$req->Y.',z='.$req->Z." for layers ".$req->layers_str);
+
+ # stop my childs
+ if(defined $self->{'rendererChildren'} ) {
+ $self->{'rendererSemaphore'}->down();
+ $self->{SHARED}->{DESTROYED} = 1;
+ $self->{'rendererSemaphore'}->up();
+ }
+ if(defined $self->{optimizePngTasks}) {
+ $self->{optimizePngTasks}->killAllChilds();
+ sleep 2; # wait of the childs
+ }
+
+
# Cleaning up of tmpdirs etc. are called in the destructor DESTROY
+ # TODO: i move it back! DESTORY is not called only one time!
+ # the GC from perl call DESTROY a 2. time and in thread mode 2*child
+ $self->cleanup();
}
sub generateNormalLayer
@@ -350,6 +519,10 @@ sub generateNormalLayer
{ # Forking to render zoom levels in parallel
$self->forkedRender($layer, $layerDataFile);
}
+ elsif ($self->{Config}->get("Cores") && defined $self->{'rendererChildren'})
+ { # use threads for rendering zoom levels parallel
+ $self->threadedRender($layer, $layerDataFile);
+ }
else
{ # Non-forking render
$self->nonForkedRender($layer, $layerDataFile);
@@ -996,6 +1169,65 @@ sub nonForkedRender
$self->Render($layer, $zoom, $layerDataFile)
}
+ if (defined $self->{optimizePngTasks}) {
+ $self->{optimizePngTasks}->wait();
+ $self->{optimizePngTasks}->dataReset();
+ }
+
+
+ if ($Config->get("CreateTilesetFile") and !$Config->get("LocalSlippymap")) {
+ $self->createTilesetFile($layer);
+ }
+ else {
+ $self->createZipFile($layer);
+ }
+}
+
+#-------------------------------------------------------------------
+# renders the tiles, not using threads
+# paramter: ($layer, $layerDataFile)
+#-------------------------------------------------------------------
+sub threadedRender
+{
+ my $self = shift;
+ my ($layer, $layerDataFile) = @_;
+ my $req = $self->{req};
+ my $Config = $self->{Config};
+ my $minzoom = $req->Z;
+ my $maxzoom = $Config->get($layer."_MaxZoom");
+
+ # access: lock()
+ $self->{'rendererSemaphore'}->down();
+
+ for (my $zoom = $maxzoom ; $zoom >= $req->Z; $zoom--) {
+
+ my $pos = $#{ $self->{SHARED}->{RENDERERJOBS} };
+ $pos++;
+
+ ::statusMessage("add renderjob zoom: $zoom at pos $pos" ,1,10);
+
+ $self->{SHARED}->{RENDERERJOBS}->[ $pos ] = $zoom;
+ $self->{SHARED}->{RENDERERJOBLAYER}->[ $pos ] = $layer;
+ $self->{SHARED}->{RENDERERJOBLAYERDATA}->[ $pos ] = $layerDataFile;
+ }
+
+ # access: unlock()
+ $self->{'rendererSemaphore'}->up();
+
+ #############
+ # at this time is the client on work and the main process wait now
+ #############
+ while($self->{SHARED}->{RENDERERJOBSREADY} <= $#{ $self->{SHARED}->{RENDERERJOBS} } ) {
+ sleep 1;
+ }
+
+ sleep 2; # dead zone
+
+ if (defined $self->{optimizePngTasks}) {
+ $self->{optimizePngTasks}->wait();
+ $self->{optimizePngTasks}->dataReset();
+ }
+
if ($Config->get("CreateTilesetFile") and !$Config->get("LocalSlippymap")) {
$self->createTilesetFile($layer);
}
@@ -1280,11 +1512,16 @@ sub SplitTiles
print $fp $png_data;
close $fp;
+ if(defined $self->{optimizePngTasks}) {
+ $self->{optimizePngTasks}->addJob($tile_file,$Config->get("${layer}_Transparent"));
+ }
+ else {
$self->optimizePng($tile_file, $Config->get("${layer}_Transparent"));
}
}
}
}
+ }
}
_______________________________________________
Tilesathome mailing list
[email protected]
http://lists.openstreetmap.org/listinfo/tilesathome
