Hello Everyone,
Thanks for the good suggestions towards the knowledge base of HDF. I am
aware of most of the stuff, but this thread also helped me to know more
about HDF.
Coming to the main point, attached is small piece of code, in which i
register a custom filter and then trying to write a dataset using the
custom filter. For some reason i get a segmentation fault after the
application i.e, DWrite completes. Any help in this regard is welcome.
Thanks
Sravan
On Wed, May 14, 2014 at 9:48 PM, Miller, Mark C. <[email protected]> wrote:
> I am sure this DOES NOT satisfy your criterion of 'simple', but if you
> download the 'Legacy Licensed' Silo 4.9.1 tarball from here,
>
> https://wci.llnl.gov/codes/silo/downloads.html
>
> untar it and go to silo-4.9.1/src/hdf5_drv and look at silo_hdf5.c and
> search for 'hzip' OR 'fpzip', you'll see how Silo does it.
>
> Mark
>
>
>
>
> From: sravan kumar <[email protected]>
> Reply-To: HDF Users Discussion List <[email protected]>
> Date: Wednesday, May 14, 2014 2:25 AM
> To: "[email protected]" <[email protected]>
> Subject: [Hdf-forum] Custom filter registration to HDF filter
>
> Hello Everyone,
>
> I am trying to register my custom compression filter with HDF library and
> create data sets using my compression technique.
>
> I am following the gzip example from the following link as reference:
>
> http://www.hdfgroup.org/HDF5/faq/compression.html
>
> Can some one point/share a different example (probably simple), which
> explains the registering of filter, creation of dataset and reading of
> dataset.
>
> --
> With Best Regards,
> Sravan Kumar
>
>
> _______________________________________________
> Hdf-forum is for HDF software users discussion.
> [email protected]
>
> http://mail.lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
>
>
--
With Best Regards,
Y. Sravan Kumar
http://meetsravan.blogspot.com/
#include "hdf5.h"
#include "RLE.h"
int main()
{
hid_t file, data_space, dataset;
hid_t rle_plist, status;
hsize_t dimsf[2];
dimsf[0] = 16;
dimsf[1] = 16;
hsize_t chunk_size[2];
chunk_size[0] = 16;
chunk_size[1] = 16;
//writing random data to inp_data
uint8_t *inp_data = new uint8_t[INP_DATA_SIZE];
for(int i=0;i<INP_DATA_SIZE;i+=10)
for( int j=i+0; j<i+10;j++)
inp_data[j]= i;
// creating an empty HDF5 file with default access/creation properties
file = H5Fcreate("rle_file.h5", H5F_ACC_TRUNC, H5P_DEFAULT,H5P_DEFAULT);
/*
* Create a property list and setting a default value to fill the dataset
*/
rle_plist = H5Pcreate(H5P_DATASET_CREATE); // Property list for using rle filter
H5Pset_chunk(rle_plist, 2, chunk_size);
/*
* Setting the new filter to our property list for creating the dataset
*/
unsigned int cd_values [1] = {0};
status = H5Pset_filter(rle_plist, RLE_FILTER_ID, H5Z_FLAG_MANDATORY, 1, cd_values);
if(status<0)
{
cout<<"failed in setting the filter\n";
return -1;
}
/*
* Create a new dataset within the file. The datatype
* and data space describe the data on disk, which may
* be different than the format used in the application's
* memory.
*/
data_space = H5Screate_simple(2, dimsf, NULL);
std::string blk_dataset = "data";
dataset = H5Dcreate (file, blk_dataset.c_str(), H5T_NATIVE_UCHAR, data_space, H5P_DEFAULT, rle_plist, H5P_DEFAULT);
cout<<"size of H5T_NATIVE_UCHAR = "<<sizeof(H5T_NATIVE_UCHAR)<<endl;
/*
* Checking if the RLE filter is available, confirming that it is registered successfully
*/
int rle_avail = H5Zfilter_avail(RLE_FILTER_ID);
unsigned int filter_config;
if(rle_avail)
{
status = H5Zget_filter_info(RLE_FILTER_ID, &filter_config);
if((filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED) && (filter_config && H5Z_FILTER_CONFIG_DECODE_ENABLED))
cout<<"rle_filter is available\n";
}
/*
* Write the array to the file. The datatype and data
* space describe the format of the data in the `dd'
* buffer. The raw data is translated to the format
* required on disk defined above. We use default raw
* data transfer properties.
*/
int err = H5Dwrite (dataset, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, H5P_DEFAULT, inp_data);
delete [] inp_data;
H5Pclose (rle_plist);
H5Dclose (dataset);
H5Sclose (data_space);
H5Fclose (file);
return 0;
}
#include "RLE.h"
/**********************************************************************************
Params: mapData = pointer to the uncompressed map.
returnData = memory location where we put the compressed map
totalData = total bytes of valid compressed data in "returnData"
Return: 0 = success
***********************************************************************************/
int32_t encodeMapData(uint8_t *inpData, uint8_t *returnData, uint32_t *totalData)
{
uint8_t *currentInpPointer = inpData;
uint8_t *currentDataPointer = returnData;
uint32_t DataConsumed = 0;
*totalData = 0;
while(DataConsumed < INP_DATA_SIZE)
{
uint8_t compressionValue = *currentInpPointer++;
uint8_t compressionCount = 0;
DataConsumed++;
while((*currentInpPointer == compressionValue) && (compressionCount < 255) && (DataConsumed < INP_DATA_SIZE))
{
compressionCount++;
currentInpPointer++;
DataConsumed++;
}
*currentDataPointer++ = compressionCount;
*currentDataPointer++ = compressionValue;
*totalData += 2;
}
printf("encodeMapData::DataConsumed = %u\n", DataConsumed);
printf("encodeMapData::totalData = %u\n", *totalData);
return 0;
}
/*
* This has declarations for RLE encode and decode functions
*/
#include <stdio.h>
#include <stdint.h>
#include <string>
#include <iostream>
#include <time.h>
#include <H5PLextern.h>
#define INP_DATA_SIZE 256
#define RLE_FILTER_ID 3008
using namespace std;
typedef unsigned long long timestamp_t;
/**********************************************************************************
Params: inpData = pointer to the uncompressed map.
returnData = memory location where we put the compressed map
totalData = total bytes of valid compressed data in "returnData"
Return: 0 = success
***********************************************************************************/
int32_t encodeMapData(uint8_t *inpData, uint8_t *returnData, uint32_t *totalData);
/*
* This code is to develop a test filter, register and use it in application.
* This has the filter function defined for RLE.
*/
#include "RLE.h"
size_t rle_filter(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf);
const H5Z_class2_t H5Z_RLE[1]=
{
{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
RLE_FILTER_ID, /* Filter id number */
1, 1, /* Encoding and decoding enabled */
"RLE_FILTER", /* Filter name for debugging */
NULL, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)rle_filter, /* The actual filter function */
}
};
H5PL_type_t H5PLget_plugin_type(void){return H5PL_TYPE_FILTER;}
const void *H5PLget_plugin_info(void){return H5Z_RLE;}
uint32_t dataSize = 0;
size_t rle_filter(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf)
{
if(flags & H5Z_FLAG_REVERSE)
{
// Decompression Logic
}
else
{
/* Compression Logic */
cout<<"********** Compress **********\n";
cout<<"flags = "<<flags<<endl;
cout<<"cd_nelmts = "<<cd_nelmts<<endl;
cout<<"nbytes = "<<nbytes<<endl;
cout<<"buf_size = "<<*buf_size<<endl;
//Allocating 2*INP_DATA_SIZE is the worst case of RLE compression
uint8_t *compressedData = new uint8_t[2*INP_DATA_SIZE];
int encode_ret = encodeMapData((uint8_t*)(*buf), compressedData, &dataSize);
cout<<"encode_ret = "<<encode_ret<<endl; // To check if encode completed successfully
cout<<"The compressed dataSize is: "<<dataSize<<endl;
// Freeing the existing memory of *buf
delete [] (uint8_t*)(*buf);
// Allocating new memory for *buf of size dataSize and copying the compressed data to it
*buf = new uint8_t[dataSize];
memcpy((uint8_t*)(*buf), compressedData, sizeof(uint8_t)*dataSize);
*buf_size = dataSize;
delete [] compressedData;
cout<<"The compressed dataSize is: "<<dataSize<<endl;
cout<<"*****************************************\n";
return dataSize;
}
return 0;
}
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://mail.lists.hdfgroup.org/mailman/listinfo/hdf-forum_lists.hdfgroup.org
Twitter: https://twitter.com/hdf5
