The attached patch adds a new mode to tilesGen.pl. If called as
"tilesGen.pl upload_loop" it will upload all ZIP files it finds in its
uploadable directory. Once they are all uploaded it will wait for new
files to arrive.
This is useful if one uses the UploadToDirectory option to have a
number of clients upload files into the same directory and a dedicated
client to upload the files onto the server.
Additionally the tests for Inkscape, pngcrush and so on are skipped if
tilesGen.pl is called in one of the upload modes. This avoids the
need to have Inkscape installed on the uploading machine.
Matthias
Index: tilesGen.pl
===================================================================
--- tilesGen.pl (revision 9137)
+++ tilesGen.pl (working copy)
@@ -55,8 +55,19 @@
$Config->args(); # overwrite config options with command line options
$Config->file("general.conf"); # overwrite with hardcoded values that must not be changed
ApplyConfigLogic($Config);
-my %EnvironmentInfo = CheckConfig($Config);
+# Handle the command-line
+my $Mode = shift();
+
+if (($Mode eq "upload") or ($Mode eq "upload_conditional") or ($Mode eq "upload_loop"))
+{
+ my %EnvironmentInfo = CheckBasicConfig($Config);
+}
+else
+{
+ my %EnvironmentInfo = CheckConfig($Config);
+}
+
my $Layers = $Config->get("Layers");
# Get version number from version-control system, as integer
@@ -120,9 +131,6 @@
# keep track of the server time for current job
my $JobTime;
-# Handle the command-line
-my $Mode = shift();
-
# If batik agent was started automatically, turn it off at exit
our $StartedBatikAgent = 0;
@@ -350,6 +358,64 @@
statusMessage("don't run this parallel to another tilesGen.pl instance", $currentSubTask, $progressJobs, $progressPercent,1);
uploadIfEnoughTiles();
}
+elsif ($Mode eq "upload_loop")
+{
+ statusMessage("don't run this parallel to another tilesGen.pl instance", $currentSubTask, $progressJobs, $progressPercent,1);
+ my $startTime = time();
+ my $elapsedTime;
+ $progressJobs = 1;
+ while(1)
+ {
+ ## before we start (another) round of rendering we first check if something bad happened in the past.
+ if (getFault("fatal") > 0)
+ {
+ cleanUpAndDie("Fatal error occurred during loop, exiting","EXIT",1,$PID);
+ }
+ elsif (getFault("upload") > 5)
+ {
+ cleanUpAndDie("Five times the upload failed, perhaps the server doesn't like us, exiting","EXIT",1,$PID);
+ }
+
+ my $sleepdelay = 1;
+ # look for stopfile and exit if found
+ if (-e "stopfile.txt")
+ {
+ cleanUpAndDie("Stopfile found, exiting","EXIT",7,$PID); ## TODO: agree on an exit code scheme for different types of errors
+ }
+
+ # Add a basic auto-updating mechanism.
+ if (NewClientVersion())
+ {
+ UpdateClient();
+ $dirent = "0/0/0"; # force reexec after update
+ }
+
+ reExecIfRequired(-1); ## check for new version of tilesGen.pl and reExec if true
+
+ if (countZips() > 0)
+ {
+ $upload_result = upload(1); # only uploading ZIP files here
+
+ if ($upload_result) # we got an error in the upload process
+ {
+ addFault("upload",1); # we only track errors that occur multple times in a row
+ }
+ else
+ {
+ resetFault("upload"); #reset fault counter for uploads if once without error
+ statusMessage("upload finished", $currentSubTask, $progressJobs, $progressPercent,1);
+ $progressJobs++;
+ }
+ $startTime = time();
+ }
+ else
+ {
+ $elapsedTime = time() - $startTime;
+ statusMessage(sprintf("waiting for new ZIP files to upload %d:%02d", $elapsedTime/60, $elapsedTime%60), $currentSubTask, $progressJobs, $progressPercent,0);
+ sleep(1);
+ }
+ }
+}
elsif ($Mode eq "version")
{
exit(1);
@@ -414,16 +480,34 @@
print " $0 loop - runs continuously\n";
print " $0 upload - uploads any tiles\n";
print " $0 upload_conditional - uploads tiles if there are many waiting\n";
+ print " $0 upload_loop - uploads tiles in loop mode\n";
print " $0 startBatik - start batik agent\n";
print " $0 stopBatik - stop batik agent\n";
print " $0 version - prints out version string and exits\n";
print "\nGNU General Public license, version 2 or later\n$Bar\n";
}
+sub countZips
+{
+ my $ZipCount = 0;
+ if (opendir(my $dp, $Config->get("WorkingDirectory")."uploadable"))
+ {
+ while(my $File = readdir($dp))
+ {
+ $ZipCount++ if ($File =~ /\.zip$/);
+ }
+ closedir($dp);
+ }
+ else
+ {
+ mkdir $Config->get("WorkingDirectory")."uploadable";
+ }
+ return $ZipCount;
+}
+
sub uploadIfEnoughTiles
{
my $Count = 0;
- my $ZipCount = 0;
# compile a list of the "Prefix" values of all configured layers,
# separated by |
@@ -444,18 +528,7 @@
mkdir $Config->get("WorkingDirectory");
}
- if (opendir(my $dp, $Config->get("WorkingDirectory")."uploadable"))
- {
- while(my $File = readdir($dp))
- {
- $ZipCount++ if ($File =~ /\.zip$/);
- }
- closedir($dp);
- }
- else
- {
- mkdir $Config->get("WorkingDirectory")."uploadable";
- }
+ my $ZipCount = countZips();
if (($Count >= 200) or ($ZipCount >= 1))
{
@@ -1876,7 +1949,7 @@
statusMessage("Waiting for child process", $currentSubTask, $progressJobs, $progressPercent,0);
waitpid($child_pid, 0);
}
- exec "perl", $0, "loop", "reexec",
+ exec "perl", $0, $Mode, "reexec",
"progressJobs=$progressJobs",
"idleSeconds=" . getIdle(1),
"idleFor=" . getIdle(0),
Index: tahconfig.pm
===================================================================
--- tahconfig.pm (revision 9137)
+++ tahconfig.pm (working copy)
@@ -61,9 +61,9 @@
}
#--------------------------------------------------------------------------
-# Checks a [EMAIL PROTECTED] configuration
+# Checks a [EMAIL PROTECTED] basic configuration
#--------------------------------------------------------------------------
-sub CheckConfig
+sub CheckBasicConfig
{
my $Config = shift();
my %EnvironmentInfo;
@@ -84,6 +84,99 @@
die ("! no subversion command set");
}
+ # LocalSplippymap
+ if ($Config->get("LocalSlippymap"))
+ {
+ print "- Writing LOCAL slippy map directory hierarchy, no uploading\n";
+ }
+ else
+ {
+ # Upload URL, username
+ printf "- Uploading with username \"".$Config->get("UploadUsername")."\"\n", ;
+ my $pw = $Config->get("UploadPassword");
+ if($pw =~ /\W/){
+ die("Check your upload password\n");
+ }
+
+ if($Config->get("UploadChunkSize") < 0.2){
+ $Config->get("UploadChunkSize") = 2;
+ print "! Using default upload chunk size of 2.0 MB\n";
+ }
+
+ if($Config->get("DeleteZipFilesAfterUpload")){
+ print "- Deleting ZIP files after upload\n";
+ }
+ }
+
+ if ($Config->get("UploadToDirectory"))
+ {
+ if (! $Config->get("UseHostnameInZipname"))
+ {
+ print " * UseHostnameInZipname should be set when using UploadToDirectory\n";
+ }
+ }
+
+ # layers
+ foreach my $layer(split(/,/, $Config->get("Layers")))
+ {
+ print "- Configured Layer: $layer\n";
+
+ if ($Config->get($layer."_MaxZoom") < 12 || $Config->get($layer."_MaxZoom") > 20)
+ {
+ print "Check $layer._MaxZoom\n";
+ }
+
+ for(my $zoom=12; $zoom<=$Config->get($layer."_MaxZoom"); $zoom++)
+ {
+ if (!defined($Config->get($layer."_Rules.$zoom")))
+ {
+ die "config option $layer._Rules.$zoom is not set";
+ }
+ if (!-f $Config->get($layer."_Rules.$zoom"))
+ {
+ die "rules file ".$Config->get($layer."_Rules.$zoom").
+ " referenced by config option $layer._Rules.$zoom ".
+ "is not present";
+ }
+ }
+
+ if (!defined($Config->get($layer."_Prefix")))
+ {
+ die "config option $layer._Prefix is not set";
+ }
+
+ # any combination of comma-separated preprocessor names is allowed
+ die "config option $layer._Preprocessor has invalid value"
+ if (grep { $_ !~ /frollo|maplint|close-areas|mercator|attribution|autocut/} split(/,/, $Config->get($layer."_Preprocessor")));
+
+ foreach my $reqfile(split(/,/, $Config->get($layer."_RequiredFiles")))
+ {
+ die "file $reqfile required for layer $layer as per config option ".
+ $layer."_RequiredFiles not found" unless (-f $reqfile);
+ }
+
+ }
+ print "* UploadConfiguredLayersOnly not set. \n Defaulting to uploading all zipfiles found, not just configured layers\n" unless defined($Config->get("UploadConfiguredLayersOnly"));
+
+ # Zip version
+ $cmd=$Config->get("Zip");
+ my $ZipV = `\"$cmd\" -v`;
+ $EnvironmentInfo{Zip}=$ZipV;
+
+ return %EnvironmentInfo;
+
+}
+
+#--------------------------------------------------------------------------
+# Checks a [EMAIL PROTECTED] configuration
+#--------------------------------------------------------------------------
+sub CheckConfig
+{
+ my $Config = shift();
+ my $cmd;
+
+ my %EnvironmentInfo = CheckBasicConfig($Config);
+
if ($Config->get("Batik"))
{
print "- Using Batik";
@@ -144,11 +237,6 @@
die "! invalid configuration setting for 'Osmarender' - allowed values are 'XSLT', 'orp'";
}
- # Zip version
- $cmd=$Config->get("Zip");
- my $ZipV = `\"$cmd\" -v`;
- $EnvironmentInfo{Zip}=$ZipV;
-
if ($ZipV eq "")
{
die("! Can't find zip (using \"".$Config->get("Zip")."\")\n");
@@ -183,45 +271,13 @@
if ( $Config->get("PngOptimizer") eq "optipng" )
{
- if ($OptipngV !~ /[Oo]pti[Pp][Nn][Gg]\s+(\d+\.\d+\.?\d*)/)
- {
- die("! Can't find OptiPNG (using \"".$Config->get("Optipng")."\")\n");
- }
- else
- {
- print "- OptiPNG version $1\n";
- }
- }
-
- # LocalSplippymap
- if ($Config->get("LocalSlippymap"))
- {
- print "- Writing LOCAL slippy map directory hierarchy, no uploading\n";
- }
- else
- {
- # Upload URL, username
- printf "- Uploading with username \"".$Config->get("UploadUsername")."\"\n", ;
- my $pw = $Config->get("UploadPassword");
- if($pw =~ /\W/){
- die("Check your upload password\n");
+ if ($OptipngV !~ /[Oo]pti[Pp][Nn][Gg]\s+(\d+\.\d+\.?\d*)/)
+ {
+ die("! Can't find OptiPNG (using \"".$Config->get("Optipng")."\")\n");
}
-
- if($Config->get("UploadChunkSize") < 0.2){
- $Config->get("UploadChunkSize") = 2;
- print "! Using default upload chunk size of 2.0 MB\n";
- }
-
- if($Config->get("DeleteZipFilesAfterUpload")){
- print "- Deleting ZIP files after upload\n";
- }
- }
-
- if ($Config->get("UploadToDirectory"))
- {
- if (! $Config->get("UseHostnameInZipname"))
+ else
{
- print " * UseHostnameInZipname should be set when using UploadToDirectory\n";
+ print "- OptiPNG version $1\n";
}
}
@@ -241,47 +297,6 @@
# print "Check MaxZoom\n";
#}
- # layers
- foreach my $layer(split(/,/, $Config->get("Layers")))
- {
- print "- Configured Layer: $layer\n";
-
- if ($Config->get($layer."_MaxZoom") < 12 || $Config->get($layer."_MaxZoom") > 20)
- {
- print "Check $layer._MaxZoom\n";
- }
-
- for(my $zoom=12; $zoom<=$Config->get($layer."_MaxZoom"); $zoom++)
- {
- if (!defined($Config->get($layer."_Rules.$zoom")))
- {
- die "config option $layer._Rules.$zoom is not set";
- }
- if (!-f $Config->get($layer."_Rules.$zoom"))
- {
- die "rules file ".$Config->get($layer."_Rules.$zoom").
- " referenced by config option $layer._Rules.$zoom ".
- "is not present";
- }
- }
-
- if (!defined($Config->get($layer."_Prefix")))
- {
- die "config option $layer._Prefix is not set";
- }
-
- # any combination of comma-separated preprocessor names is allowed
- die "config option $layer._Preprocessor has invalid value"
- if (grep { $_ !~ /frollo|maplint|close-areas|mercator|attribution|autocut/} split(/,/, $Config->get($layer."_Preprocessor")));
-
- foreach my $reqfile(split(/,/, $Config->get($layer."_RequiredFiles")))
- {
- die "file $reqfile required for layer $layer as per config option ".
- $layer."_RequiredFiles not found" unless (-f $reqfile);
- }
-
- }
- print "* UploadConfiguredLayersOnly not set. \n Defaulting to uploading all zipfiles found, not just configured layers\n" unless defined($Config->get("UploadConfiguredLayersOnly"));
return %EnvironmentInfo;
}
_______________________________________________
Tilesathome mailing list
[email protected]
http://lists.openstreetmap.org/cgi-bin/mailman/listinfo/tilesathome
