Quincey
I was surprised as well, especially by how much slower it was. I tried
changing to 1.8.3 when it came out but found that my application was a
lot slower, I didn't have the time then to investigate further and I
went back to 1.6.5. I have now found the same problem with 1.8.7.
My application uses HDF via a library of wrapper rountines. I wrote a
timing program using this library that mimicked the application and
found that it was about 500 times slower with 1.8.7. When I removed the
code that created/used attributes the times for 1.6.5 and 1.8.7 were
similar. I have cut and pasted bits from my library into some code I
can send you, 1.8.7 is about 40 times slower for this code -- the use of
datasets and attributes in this example are much simpler than in my
timing program or application. The code contains a call to the function
write_attribute, if this call is removed the times for 1.6.5 and 1.8.7
are similar.
I suspect that I've misused/misunderstood HDF in some way but got away
with it for 1.6.5.
Thanks for your help
Rod
On 23/06/2011 16:50, Quincey Koziol wrote:
Hi Rod,
On Jun 23, 2011, at 6:09 AM, Rod Cook wrote:
I am currently changing from 1.6.5 to 1.8.7 and my application is 500 times
slower with 1.8.7. I have narrowed the cause down to the creation (and use)
of attributes. The application that can generate 1000's of small attributes
(references to a dataset) attached to a single object. I would be grateful for
any help and advice.
Hmm, I'm surprised that things got slower in this area. Can you send a
simple C program that works with both versions and demonstrates the slowdown?
Quincey
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
/*
* Define the preprocessor symbol ZZZ_USE_HDF18 for HDF 1.8.x
*/
#include <stdlib.h>
#include <time.h>
#include "hdf5.h"
hid_t ZZ_Record_id;
typedef struct ZZZ_Index_t
{
hid_t index;
hid_t handle;
hid_t by_name;
hid_t by_id;
} ZZZ_Index;
typedef struct Internal_Transfer_t
{
ZZZ_Index index;
hsize_t first_element;
hsize_t last_element;
int abcnum;
int xyznum;
char name[100];
} Internal_Transfer;
typedef struct ZZ_Record_HDF_
{
int fields[14];
char name[1024];
} ZZ_Record_HDF;
typedef struct ZZZ_Library_t
{
hid_t handle;
hid_t dataset;
hid_t index;
int number_records;
int dataset_length;
ZZZ_Index any_index;
} ZZZ_Library;
hid_t create_ZZ_Record_id( int name_length )
{
hid_t string_type;
hid_t id, fields_type;
hsize_t array_dim[] = {14};
string_type = H5Tcopy( H5T_C_S1 );
H5Tset_size( string_type, name_length );
#ifdef ZZZ_USE_HDF18
fields_type = H5Tarray_create( H5T_NATIVE_INT, 1, array_dim );
#else
fields_type = H5Tarray_create( H5T_NATIVE_INT, 1, array_dim, NULL );
#endif
id = H5Tcreate(H5T_COMPOUND, sizeof(ZZ_Record_HDF) );
H5Tinsert( id, "fields", HOFFSET( ZZ_Record_HDF, fields ), fields_type );
H5Tinsert( id, "name", HOFFSET( ZZ_Record_HDF, name), string_type );
return id;
}
void create_index_group( const char * name, ZZZ_Index * sub_index )
{
#ifdef ZZZ_USE_HDF18
sub_index->handle = H5Gcreate( sub_index->index, name, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT );
sub_index->by_name = H5Gcreate( sub_index->handle, "By_Name", H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT );
sub_index->by_id = H5Gcreate( sub_index->handle, "By_ID", H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT );
#else
sub_index->handle = H5Gcreate( sub_index->index, name, 0 );
sub_index->by_name = H5Gcreate( sub_index->handle, "By_Name", 0 );
sub_index->by_id = H5Gcreate( sub_index->handle, "By_ID", 0 );
#endif
}
void create_dataset( ZZZ_Library * library, char * name, hid_t id, void * value
)
{
hid_t dataspace;
hid_t cparms;
hsize_t maxdim[1] = {H5S_UNLIMITED};
hsize_t dim[1];
hsize_t chunk_dims[1];
herr_t status;
dim[0] = 100;
chunk_dims[0] = 10;
dataspace = H5Screate_simple( 1, dim, maxdim );
cparms = H5Pcreate ( H5P_DATASET_CREATE );
status = H5Pset_chunk ( cparms, 1, chunk_dims );
status = H5Pset_deflate( cparms, 6 );
status = H5Pset_fill_value( cparms, id, value );
#ifdef ZZZ_USE_HDF18
library->dataset = H5Dcreate( library->handle, name, id, dataspace,
H5P_DEFAULT, cparms, H5P_DEFAULT );
#else
library->dataset = H5Dcreate( library->handle, name, id, dataspace, cparms );
#endif
library->dataset_length = 0;
library->number_records = 0;
}
void ZZZ_write_record ( ZZ_Record_HDF * ZZrec, ZZZ_Library * library)
{
int i, num_records;
hid_t dataspace, filespace;
herr_t status;
hsize_t offset[1];
hsize_t size[] = {0};
hsize_t dim1[] = {1};
filespace = H5Dget_space ( library->dataset );
H5Sget_simple_extent_dims( filespace, size, NULL );
num_records = 1;
if ( (library->dataset_length + num_records) > size[0] )
{
hsize_t min_increment;
min_increment = (library->dataset_length + num_records) - size[0];
if ( min_increment < 10 )
{
min_increment = 10;
}
size[0] = size[0] + min_increment;
status = H5Dextend (library->dataset, size);
H5Sclose( filespace );
filespace = H5Dget_space ( library->dataset );
}
dataspace = H5Screate_simple( 1, dim1, NULL );
for ( i=0;i<num_records;i++)
{
offset[0] = library->dataset_length;
status = H5Sselect_hyperslab ( filespace, H5S_SELECT_SET, offset, NULL,
dim1, NULL );
status = H5Dwrite( library->dataset, ZZ_Record_id, dataspace, filespace,
H5P_DEFAULT, &ZZrec[i] );
library->dataset_length = library->dataset_length + 1;
}
library->number_records = library->number_records + 1;
H5Sclose( filespace );
H5Sclose( dataspace );
return;
}
void write_attribute( const Internal_Transfer * index_info,
ZZZ_Library * library )
{
hid_t attr = 0, aid = 0;
hid_t filespacer = 0;
hdset_reg_ref_t ref = {0};
hsize_t index_offset = 0, index_size = 0;
herr_t status = 0;
filespacer = H5Dget_space ( library->dataset );
index_offset = index_info->first_element;
index_size = index_info->last_element - index_info->first_element + 1;
status = H5Sselect_hyperslab( filespacer, H5S_SELECT_SET, &index_offset,
NULL, &index_size, NULL );
status = H5Rcreate( &ref, library->handle, "ZZ_Records", H5R_DATASET_REGION,
filespacer );
H5Sclose( filespacer );
aid = H5Screate( H5S_SCALAR );
#ifdef ZZZ_USE_HDF18
attr = H5Acreate( index_info->index.by_name, index_info->name,
H5T_STD_REF_DSETREG, aid, H5P_DEFAULT, H5P_DEFAULT );
#else
attr = H5Acreate( index_info->index.by_name, index_info->name,
H5T_STD_REF_DSETREG, aid, H5P_DEFAULT );
#endif
status = H5Awrite( attr, H5T_STD_REF_DSETREG, &ref );
H5Sclose( aid );
H5Aclose( attr );
if (index_info->abcnum != -1 )
{
char temp[100] = {0}, id_name[100]={0}, fgid_name[100]={0};
hid_t fgid;
sprintf( fgid_name, "%d\0", index_info->abcnum );
sprintf( id_name, "%d\0", index_info->abcnum );
strcat( id_name, "--" );
if ( index_info->xyznum != -1 )
{
sprintf( temp, "%d\0", index_info->xyznum );
strcat( id_name, temp );
}
else
{
strcat( id_name, index_info->name );
}
#ifdef ZZZ_USE_HDF18
fgid = H5Gopen( index_info->index.by_id, fgid_name, H5P_DEFAULT );
#else
fgid = H5Gopen( index_info->index.by_id, fgid_name );
#endif
if ( fgid < 0 )
{
#ifdef ZZZ_USE_HDF18
fgid = H5Gcreate( index_info->index.by_id, fgid_name, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT );
#else
fgid = H5Gcreate( index_info->index.by_id, fgid_name, 0 );
#endif
}
aid = H5Screate( H5S_SCALAR );
#ifdef ZZZ_USE_HDF18
attr = H5Acreate( fgid, id_name, H5T_STD_REF_DSETREG, aid, H5P_DEFAULT,
H5P_DEFAULT );
#else
attr = H5Acreate( fgid, id_name, H5T_STD_REF_DSETREG, aid, H5P_DEFAULT );
#endif
status = H5Awrite( attr, H5T_STD_REF_DSETREG, &ref );
H5Gclose( fgid );
H5Sclose( aid );
H5Aclose( attr );
}
}
int main (void)
{
FILE *fp;
ZZ_Record_HDF rec = {0};
ZZZ_Library library;
ZZ_Record_HDF ZZrec_fill_value = {0};
Internal_Transfer index_info;
int i;
hid_t file;
clock_t start, finish;
fp = fopen( "timings.dat", "w" );
H5open( );
#ifdef ZZZ_USE_HDF18
H5Eset_auto(H5E_DEFAULT, NULL, NULL );
#else
H5Eset_auto( NULL, NULL );
#endif
ZZ_Record_id = create_ZZ_Record_id( 1001 );
file = H5Fcreate( "test.hdf", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT );
#ifdef ZZZ_USE_HDF18
library.handle = H5Gcreate( file, "library", H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT );
library.index = H5Gcreate( library.handle, "Index", H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT );
#else
library.handle = H5Gcreate( file, "library", 0 );
library.index = H5Gcreate( library.handle, "Index", 0 );
#endif
library.any_index.index = library.index;
create_index_group( "Any", &library.any_index );
create_dataset( &library, "ZZ_Records", ZZ_Record_id, &ZZrec_fill_value);
index_info.index = library.any_index;
start = clock();
for(i=0;i<1000;i++)
{
rec.fields[1] = i;
ZZZ_write_record(&rec,&library);
index_info.first_element=i;
index_info.last_element=i;
sprintf(index_info.name,"name_%d",i);
index_info.abcnum = i;
index_info.xyznum = 1000+i;
// write_attribute(&index_info, &library);
}
finish = clock();
fprintf( fp, " Time : %d\n", (finish - start));
fclose(fp);
return 0;
}
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
