Hi, I am testing in Windows XP compiling with Visual Studio 2010.
I have a console test application using directly the "shapefil" library.
I will use test_ogrsf utility for a complete benchmark
I this environment and my test application, using OGRFastAtof(), the full read
of a huge shape around 500.000 records (500mb of shp file, 150mb of dbf file)
with eight double fields goes from 5.6sg to 3.06sg. I don't evaluate that the
precision loss, it is a task for check.
In addition, using filemapping with boost library, the full process goes from
3.06sg to 1.45sg.
These values as you see are approximate, but they give an idea of
optimization is possible at least for this OS.
It is why I wanted to check this behavior
thanks!
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// TEST BOOST - MEMORY MAPPING FILE
#include <boost/filesystem.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#define ROOT_LIB_PATH( __RELATIVE_PATH ) "/TFS_OSGeo/SIT/SIG_SDK/Open
Source/boost-1.54.0"#__RELATIVE_PATH
#if _DEBUG
#pragma comment(lib, ROOT_LIB_PATH(
"/lib32-msvc-10.0/libboost_iostreams-vc100-mt-sgd-1_54.lib" ))
#pragma comment(lib, ROOT_LIB_PATH(
"/lib32-msvc-10.0/libboost_filesystem-vc100-mt-sgd-1_54.lib" ))
#else
#pragma comment(lib, ROOT_LIB_PATH(
"/lib32-msvc-10.0/libboost_iostreams-vc100-mt-s-1_54.lib" ))
#pragma comment(lib, ROOT_LIB_PATH(
"/lib32-msvc-10.0/libboost_filesystem-vc100-mt-s-1_54.lib" ))
#endif
//-------------------------------------------------------------------------------------------------------
// Helper wrapper class for boost::mapped_file.
class mapped_file_manager
{
public:
mapped_file_manager() : file(NULL), length(0), tell(0)
{
}
~mapped_file_manager()
{
close();
}
public:
boost::iostreams::mapped_file* file;
uintmax_t length;
uintmax_t tell;
public:
bool open(const char *pszFilename, bool readOnly)
{
if (boost::filesystem::exists(pszFilename))
{
boost::iostreams::mapped_file* fileP = new
boost::iostreams::mapped_file();
fileP->open(pszFilename, readOnly ?
boost::iostreams::mapped_file_source::readwrite :
boost::iostreams::mapped_file_source::readwrite);
if (fileP->is_open())
{
length =
boost::filesystem::file_size(pszFilename);
file = fileP;
return true;
}
}
return false;
}
bool close()
{
if (file)
{
length = 0;
tell = 0;
file->close();
delete file;
file = NULL;
return true;
}
return false;
}
};
SAFile MMF_SHP_Open( const char *pszFilename, const char *pszAccess )
{
if (boost::filesystem::exists(pszFilename))
{
mapped_file_manager* manager = new mapped_file_manager();
if (manager->open(pszFilename, strstr(pszAccess, "r")!=NULL))
{
#if _DEBUG
size_t size = manager->file->size();
#endif
return (SAFile)manager;
}
delete manager;
}
return NULL;
}
SAOffset MMF_SHP_Read( void *p, SAOffset size, SAOffset nmemb, SAFile file )
{
mapped_file_manager* manager = (mapped_file_manager*)file;
char* data = manager->file->data();
int memoyrSize = size*nmemb;
memcpy(p, data+manager->tell, memoyrSize);
manager->tell += memoyrSize;
return nmemb;
}
SAOffset MMF_SHP_Write( void *p, SAOffset size, SAOffset nmemb, SAFile file )
{
mapped_file_manager* manager = (mapped_file_manager*)file;
char* data = manager->file->data();
int memoyrSize = size*nmemb;
memcpy(data+manager->tell, p, memoyrSize);
manager->tell += memoyrSize;
return nmemb;
}
SAOffset MMF_SHP_Seek( SAFile file, SAOffset offset, int whence )
{
mapped_file_manager* manager = (mapped_file_manager*)file;
if (whence==SEEK_SET)
{
manager->tell = offset;
return 0;
}
else
if (whence==SEEK_END)
{
manager->tell = manager->length+offset;
return offset<0 ? 0 : 1;
}
else
if (whence==SEEK_CUR)
{
manager->tell = manager->tell+offset;
return 0;
}
return 2;
}
SAOffset MMF_SHP_Tell( SAFile file )
{
mapped_file_manager* manager = (mapped_file_manager*)file;
return (SAOffset)manager->tell;
}
int MMF_SHP_Flush( SAFile file )
{
mapped_file_manager* manager = (mapped_file_manager*)file;
// TODO:
return 0;
}
int MMF_SHP_Close( SAFile file )
{
if (file)
{
mapped_file_manager* manager = (mapped_file_manager*)file;
manager->close();
delete manager;
return 1;
}
return 0;
}
void MMF_SHP_Error( const char *message )
{
CPLError( CE_Failure, CPLE_AppDefined, "%s", message );
}
int MMF_SHP_Remove( const char *pszFilename )
{
return VSIUnlink( pszFilename );
}
//-------------------------------------------------------------------------------------------------------
void SASetupBoostlibHooks( SAHooks *psHooks )
{
psHooks->FOpen = MMF_SHP_Open;
psHooks->FRead = MMF_SHP_Read;
psHooks->FWrite = MMF_SHP_Write;
psHooks->FSeek = MMF_SHP_Seek;
psHooks->FTell = MMF_SHP_Tell;
psHooks->FFlush = MMF_SHP_Flush;
psHooks->FClose = MMF_SHP_Close;
psHooks->Remove = MMF_SHP_Remove;
psHooks->Atof = DBFFastAtof;
psHooks->Error = MMF_SHP_Error;
}
//-------------------------------------------------------------------------------------------------------
_______________________________________________
Qgis-developer mailing list
Qgis-developer@lists.osgeo.org
http://lists.osgeo.org/mailman/listinfo/qgis-developer
