Michael Ritzert wrote:
Hi spaetz,
spaetz wrote:
I had started writing an apache module in C for the tileset stuff before
I heard that Spaetz rewrote the entire server code. If it is helpful, I
would be happy to continue on it again and adapt it to the new server
layout. It would probably take a couple of days though till I find the
time to finish of the testing. Please let me know if other people are
working on it or if the memory consumption is no longer an issue to
prevent duplicate effort.
I have done a (not so) quick (my last C coding was years ago) and ugly
fastcgi in ANSI C to serve tiles. And given that the [EMAIL PROTECTED] server
has just
died again, I will try to use it ASAP when the server is up again.
I had a quick look at it and found that ripping everything from mod_example.c
shipped with apache and implementing serving tiles (via sendfile, so no copy to
userspace needed) is fairly easy. I have the stripped mod_example.c available,
so if I can get my hands on your FastCGI C code, I may copy the relavant parts
to the apache module. Is it availably in the SVN?
I have written a version of mod_tah and attached it to this mail. It is
not particularly well tested (nor properly documented), but I hope it
would work and has both the fall-back and the oceantiles lookup included.
Looking at apachebench, it seems roughly double the speed compared to
the "old" mod_python serving. Trying to asses the memory consumption, it
seems that apache processes dropped from about 14Mb to 7Mb of resident
memory, but then there were not many processes at any time, so the
overall saving doesn't seem significant. All of this is hitting the
same tile each time, so there is no disk IO involved, which may change
this all completely.
I guess there were talks of moving over completely to lighthttpd, so I
don't know if this implementation will ever be used, but I thought I'd
just post it here for reference in case it might become useful. Also I
don't know if it is any better than the fastcgi version.
Kai
Michael
_______________________________________________
Tilesathome mailing list
[email protected]
http://lists.openstreetmap.org/cgi-bin/mailman/listinfo/tilesathome
#include "httpd.h"
#include "http_config.h"
#include "http_protocol.h"
#include "http_log.h"
#include "ap_config.h"
#include "apr_strings.h"
#include <apr_file_info.h>
#include <apr_file_io.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
module AP_MODULE_DECLARE_DATA tilesAtHome_module;
#define FILEVERSION 1
#define MIN_VALID_OFFSET 4
static char * basetilepath = "/usr/local/src/tah/Tiles";
static char * legacytilepath = "/var/www/tilesAtHome/Tiles";
#define OCEANS_DB_FILE "/var/www/tilesAtHome/oceantiles_12.dat"
#define UNKNOWN_TILE_FILENAME "/var/www/tilesAtHome/Tiles/unknown.png"
#define LAND_TILE_FILENAME "/var/www/tilesAtHome/Tiles/land.png"
#define SEA_TILE_FILENAME "/var/www/tilesAtHome/Tiles/sea.png"
#define TRANSPARENT_TILE_FILENAME "/var/www/tilesAtHome/Tiles/transparent.png"
#define ERROR_TILE_FILENAME "/var/www/tilesAtHome/Tiles/error.png"
static void basexyz_to_tilesetname(apr_pool_t *p, char ** tilesetName, char * layer, int x, int y, int z) {
char * fileName;
fileName = apr_psprintf(p, "%s/%s_%02d/%i/%i_%i",basetilepath, layer, z, x, x, y);
*tilesetName = fileName;
};
static void xyz_to_basexyz(int x, int y, int z, int * baseX, int * baseY, int * baseZ) {
if (z < 6) {
*baseZ = 0;
} else if ( z > 11) {
*baseZ = 12;
} else {
*baseZ = 6;
}
*baseX = x >> (z - *baseZ);
*baseY = y >> (z - *baseZ);
}
static int xyz_to_n(int x, int y, int z, int baseX, int baseY, int baseZ, request_rec * r) {
int offsetX;
int offsetY;
int tileNo = 0;
int i;
for (i = baseZ; i < z; i++) {
tileNo += 1 << (2*(i - baseZ));
}
offsetX = baseX << (z - baseZ);
offsetY = baseY << (z - baseZ);
tileNo += (y - offsetY) * (1 << (z - baseZ));
tileNo += (x - offsetX);
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "baseZ %i OffsetX %i OffsetY %i tileNo %i", baseZ, offsetX, offsetY, tileNo);
return tileNo;
}
static void xyz_to_legacytile(request_rec *r, char ** tilesetName, char * layer, int x, int y, int z) {
char * fileName;
fileName = apr_psprintf(r->pool, "%s/%s/%02d/%03d/%03d/%03d/%03d.png",legacytilepath, layer, z, (x/1000), (x%1000), (y/1000), (y%1000));
*tilesetName = fileName;
}
static int xyz_to_blankdbtile(request_rec *r, char ** tileName, char * layer, int x, int y, int z, int baseX, int baseY) {
char * fileName;
apr_file_t * oceandb;
apr_status_t res;
apr_size_t len;
apr_off_t offset;
unsigned char data[4];
int bit_off;
int type;
if (z < 12) {
fileName = apr_pstrdup(r->pool, UNKNOWN_TILE_FILENAME);
} else {
if ((res = apr_file_open(&oceandb, OCEANS_DB_FILE, APR_READ, APR_OS_DEFAULT, r->pool)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "ERROR: failed to open Oceans DB %s", fileName);
return 1;
};
offset = (4096*baseY + baseX) >> 2;
// ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Oceantiles baseX %i baseY %i offset %i", baseX, baseY, offset);
if (apr_file_seek(oceandb, APR_SET, &offset) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Oceantiles is CORRUPT, ");
return HTTP_NOT_FOUND;
}
len = 1;
if ((res = apr_file_read(oceandb, data, &len)) != APR_SUCCESS){
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Oceantiles is CORRUPT, ");
return HTTP_NOT_FOUND;
}
bit_off = 3 - (baseX % 4);
type = ((data[0] >> (2*bit_off)) & 3);
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Oceantiles bit_off %i data %i type %i", bit_off, data[0], type);
switch (type) {
case 0 : {
fileName = apr_pstrdup(r->pool, UNKNOWN_TILE_FILENAME);
break;
}
case 1 : {
fileName = apr_pstrdup(r->pool, LAND_TILE_FILENAME);
break;
}
case 2 : {
fileName = apr_pstrdup(r->pool, SEA_TILE_FILENAME);
break;
}
case 3 : {
fileName = apr_pstrdup(r->pool, UNKNOWN_TILE_FILENAME);
break;
}
}
apr_file_close(oceandb);
}
*tileName = fileName;
return 0;
}
static int tah_translate(request_rec *r)
{
int x, y, z, n, limit;
apr_status_t res;
char layer[32];
char * tilesetName;
int baseX, baseY, baseZ;
int tileOffset, tileSize;
struct apr_finfo_t finfo;
apr_file_t * tileset;
apr_size_t len;
apr_off_t offset;
int buf[32];
layer[0] = '\0';
// ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "translate handler(%s), uri(%s), filename(%s), path_info(%s)",
// r->handler, r->uri, r->filename, r->path_info);
/* URI = ...Tiles/[layer]/<z>/<x>/<y>.png */
n = sscanf(r->uri, "/tilesAtHome/Tiles/%31[a-z]/%d/%d/%d.png", layer, &z, &x, &y);
if (n < 4)
return DECLINED;
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "layer(%s) z(%d) x(%d) y(%d)", layer, z, x, y);
xyz_to_basexyz(x,y,z,&baseX, &baseY, &baseZ);
basexyz_to_tilesetname(r->pool, &tilesetName, layer,baseX,baseY,baseZ);
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile: %s", tilesetName);
if ((res = apr_stat(&finfo, tilesetName, APR_FINFO_MTIME, r->pool)) != APR_SUCCESS) {
xyz_to_legacytile(r, &tilesetName, layer, x, y, z);
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "No tilesetfile, trying legacy tile: %s", tilesetName);
if ((res = apr_stat(&finfo, tilesetName, APR_FINFO_MTIME, r->pool)) != APR_SUCCESS) {
if (xyz_to_blankdbtile(r, &tilesetName, layer, x, y, z, baseX, baseY) != 0) {
return HTTP_NOT_FOUND;
};
}
//Use the standard apache fileserving mechanism to serve these tiles
//that are a file per tile rather than in a tileset
r->filename = apr_pstrdup(r->pool, tilesetName);
return OK;
}
ap_update_mtime(r, finfo.mtime);
ap_set_last_modified(r);
apr_table_setn(r->headers_out, "Cache-Control","max-age=10800, must-revalidate");
if ((res = ap_meets_conditions(r)) != OK) {
return res;
}
if ((res = apr_file_open(&tileset, tilesetName, APR_READ, APR_OS_DEFAULT, r->pool)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "ERROR: failed to open tilesetfile");
//this shouldn't happen, as we checked the file before;
return HTTP_NOT_FOUND;
};
len = 2*sizeof(int);
if (((res = apr_file_read(tileset, buf, &len)) != APR_SUCCESS) && ((*((int *)(&buf[0]))) != FILEVERSION)){
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile header %s is CORRUPT", tilesetName);
return HTTP_NOT_FOUND;
}
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "baseX %i baseY %i, int: %i", baseX, baseY, sizeof(int));
int tileNo = xyz_to_n(x,y,z, baseX, baseY, baseZ, r);
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tile number %i", tileNo);
offset = 2*sizeof(int) + tileNo*sizeof(int);
if (apr_file_seek(tileset, APR_SET, &offset) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT, ", tilesetName);
return HTTP_NOT_FOUND;
}
len = sizeof(int);
if ((res = apr_file_read(tileset, buf, &len)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT could not read offset, ", tilesetName);
}
apr_file_close(tileset);
tileOffset = buf[0];
if (tileOffset < 0) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT! Negative offset ", tilesetName);
return HTTP_NOT_FOUND;
}
if (tileOffset < MIN_VALID_OFFSET) {
switch (tileOffset) {
case 0: {
r->filename = apr_pstrdup(r->pool, UNKNOWN_TILE_FILENAME);
break;
}
case 1: {
r->filename = apr_pstrdup(r->pool, SEA_TILE_FILENAME);
break;
}
case 2: {
r->filename = apr_pstrdup(r->pool, LAND_TILE_FILENAME);
break;
}
case 3: {
r->filename = apr_pstrdup(r->pool, TRANSPARENT_TILE_FILENAME);
break;
}
default:{
r->filename = apr_pstrdup(r->pool, ERROR_TILE_FILENAME);
break;
}
}
} else {
r->filename = apr_pstrdup(r->pool, tilesetName);
r->handler = "tah_serve";
}
return OK;
}
static int tah_handler_serve(request_rec *r)
{
int x, y, z, n, limit, oob;
int baseX, baseY, baseZ;
int buf[64];
char * data;
apr_status_t res;
apr_file_t * tileset;
apr_size_t len;
apr_off_t offset;
int tileOffset, tileLength;
char layer[32];
// ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "serve handler(%s), uri(%s), filename(%s), path_info(%s)",
// r->handler, r->uri, r->filename, r->path_info);
if(strcmp(r->handler, "tah_serve"))
return DECLINED;
/* URI = ...Tiles/[layer]/<z>/<x>/<y>.png */
n = sscanf(r->uri, "/tilesAtHome/Tiles/%31[a-z]/%d/%d/%d.png", layer, &z, &x, &y);
if ((res = apr_file_open(&tileset, r->filename, APR_READ, APR_OS_DEFAULT, r->pool)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "ERROR: failed to open tilesetfile");
//this shouldn't happen, as we checked the file before;
return HTTP_NOT_FOUND;
};
xyz_to_basexyz(x,y,z,&baseX, &baseY, &baseZ);
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "X %i Y %i Z %i baseZ %i baseY %i baseX %i ", x, y, z, baseZ, baseY, baseX);
int tileNo = xyz_to_n(x,y,z, baseX, baseY, baseZ, r);
offset = 2*sizeof(int) + tileNo*sizeof(int);
if (apr_file_seek(tileset, APR_SET, &offset) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT, ", r->filename);
return HTTP_NOT_FOUND;
}
len = 32*sizeof(int);
if ((res = apr_file_read(tileset, buf, &len)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT could not read offset, ", r->filename);
}
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "buf %i %i %i %i", buf[0], buf[1], buf[2], buf[3]);
tileOffset = buf[0];
if (tileOffset < MIN_VALID_OFFSET) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Got a negative tile offset in handler");
return HTTP_NOT_FOUND;
}
int i = 1;
tileLength = buf[i];
while (tileLength < MIN_VALID_OFFSET) {
i++;
if (i > 31) {
len = 32*sizeof(int);
if ((res = apr_file_read(tileset, buf, &len)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT could not read offset, ", r->filename);
}
i = 0;
}
tileLength = buf[i];
}
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile %s offset % i, length %i, %i %i", r->filename, tileOffset, tileLength, i, sizeof(int));
tileLength -= tileOffset;
//ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile %s offset % i, length %i", r->filename, tileOffset, tileLength);
ap_set_content_length(r, tileLength);
ap_set_content_type(r, "image/png");
data = malloc(tileLength + 10);
offset = tileOffset;
if (apr_file_seek(tileset, APR_SET, &offset) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile index %s is CORRUPT, ", r->filename);
free(data);
return HTTP_NOT_FOUND;
}
len = tileLength;
if ((res = apr_file_read(tileset, data, &len)) != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "tilesetfile CORRUPT could not read png, ", r->filename);
free(data);
return HTTP_NOT_FOUND;
}
ap_rwrite(data,tileLength,r);
free(data);
apr_file_close(tileset);
return OK;
}
static void tah_register_hooks(apr_pool_t *p) {
ap_hook_translate_name(tah_translate, NULL, NULL, APR_HOOK_FIRST);
ap_hook_handler(tah_handler_serve, NULL, NULL, APR_HOOK_MIDDLE);
}
// API hooks
module AP_MODULE_DECLARE_DATA tilesAtHome_module = {
STANDARD20_MODULE_STUFF,
NULL, /* create per-dir config structures */
NULL, /* merge per-dir config structures */
NULL, /* create per-server config structures */
NULL, /* merge per-server config structures */
NULL, /* table of config file commands */
tah_register_hooks /* register hooks */
};
_______________________________________________
Tilesathome mailing list
[email protected]
http://lists.openstreetmap.org/cgi-bin/mailman/listinfo/tilesathome
