Hi Quincey,
Here I attached my code for the benchmark.
Thank you
regards,
Elisa
--- On Tue, 9/14/10, Quincey Koziol <[email protected]> wrote:
From: Quincey Koziol <[email protected]>
Subject: Re: [Hdf-forum] Store data in hierarchy, does it impact the
performance?
To: "HDF Users Discussion List" <[email protected]>
Date: Tuesday, September 14, 2010, 6:23 PM
Hi Elisa,
On Sep 15, 2010, at 12:09 AM, elisa sibarani wrote:
> Hi Quincey,
>
> Here I attached the example of HDF5 file with hierarchy, file with no
> hierarchy inside, and the benchmark result.
Actually, I meant source code for the benchmark. Can you send that?
Quincey
> Thank you for the reply.
>
> Regards,
> Elisa MS
>
>
> --- On Tue, 9/14/10, Quincey Koziol <[email protected]> wrote:
>
> From: Quincey Koziol <[email protected]>
> Subject: Re: [Hdf-forum] Store data in hierarchy, does it impact the
> performance?
> To: "HDF Users Discussion List" <[email protected]>
> Date: Tuesday, September 14, 2010, 5:47 PM
>
> Hi Elisa,
>
> On Sep 14, 2010, at 4:43 PM, elisa sibarani wrote:
>
> > Hi All,
> >
> > I really need your help or idea, do I need to store data in hierarchy if I
> > want to use HDF5? When I do a small benchmark, the performance of the file
> > decrease when the data store directly in a dataset (after a 'root' group),
> > rather than in a hierarchical way, Is there any reason behind that result?
> >
> > Please, really need the reason of this question.
>
> Hmm, I don't have any good reason why this should be so. Do you have a
>small benchmark that demonstrates the issue?
>
> Quincey
>
>
> _______________________________________________
> Hdf-forum is for HDF software users discussion.
> [email protected]
> http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
>
>
> <BenchmarkResult.jpg><HDF_NoHierarchy.jpg><HDF_WithHierarchy.jpg>_______________________________________________
> Hdf-forum is for HDF software users discussion.
> [email protected]
> http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
private void InsertToHDF5_WithHierarchy(String circuitid, string[] groupname,
struct1[] dset_data, int numline, out int totalpd, out TimeSpan
elapsed_time_insert)
{
int countpd = 0;
bool filexist = CheckFile();
if (filexist == true)
{
fileId = H5F.open(H5File_Name, H5F.OpenMode.ACC_RDWR);
}
// Create a circuit group
// check if that group is already exist
bool circuitexist = CheckGroup(circuitid);
if (circuitexist == true)
{
//MessageBox.Show("Successfully open circuit group");
groupcircuitId = H5G.open(fileId, "/" + circuitid);
}
else if (circuitexist == false)
{
//MessageBox.Show("Successfully create circuit group");
groupcircuitId = H5G.create(fileId, "/" + circuitid);
}
//create year subgroup but check it first
bool yearexist = CheckGroup(circuitid + "/" + groupname[0]);
if (yearexist == true)
{
//MessageBox.Show("Successfully open year subgroup");
groupyearId = H5G.open(fileId, "/" + circuitid + "/" +
groupname[0]);
}
else if (yearexist == false)
{
//MessageBox.Show("Successfully create year subgroup");
groupyearId = H5G.create(fileId, "/" + circuitid + "/" +
groupname[0]);
}
//create month subgroup but check it first
bool monthexist = CheckGroup(circuitid + "/" + groupname[0] + "/" +
groupname[1]);
if (monthexist == true)
{
//MessageBox.Show("Successfully open month subgroup");
groupmonthId = H5G.open(fileId, "/" + circuitid + "/" +
groupname[0] + "/" + groupname[1]);
}
else if (monthexist == false)
{
///MessageBox.Show("Successfully create month subgroup");
groupmonthId = H5G.create(fileId, "/" + circuitid + "/" +
groupname[0] + "/" + groupname[1]);
}
//create date subgroup but check it first
bool datexist = CheckGroup(circuitid + "/" + groupname[0] + "/" +
groupname[1] + "/" + groupname[2]);
if (datexist == true)
{
//MessageBox.Show("Successfully open date subgroup");
groupdateId = H5G.open(fileId, "/" + circuitid + "/" +
groupname[0] + "/" + groupname[1] + "/" + groupname[2]);
}
else if (datexist == false)
{
//MessageBox.Show("Successfully create date subgroup");
groupdateId = H5G.create(fileId, "/" + circuitid + "/" +
groupname[0] + "/" + groupname[1] + "/" + groupname[2]);
}
//create attribute type
attributeType = H5T.copy(H5T.H5Type.NATIVE_INT);
//create attribute data space
ulong[] attributedims = new ulong[1];
attributedims[0] = 1;
int[] readBackAttr = new int[1];
attributeSpace = H5S.create_simple(1, attributedims);
//create attribute in the subgroup date
bool attributexist = CheckAttribute(groupdateId, "totaline");
if (attributexist == true)
{
//MessageBox.Show("Successfully open attribute");
attributeId = H5A.open(groupdateId, "/totaline");
}
else if (attributexist == false)
{
//MessageBox.Show("Successfully create attribute");
attributeId = H5A.create(groupdateId, "/totaline",
attributeType, attributeSpace);
}
H5A.read<int>(attributeId, attributeType, new
H5Array<int>(readBackAttr));
//Create a data space to accomodate our 1 dimensional array
//The resulting H5DataSpaceId will be used to create the dataset
ulong[] dims = new ulong[1];
dims[0] = ulong.Parse(numline.ToString());
spaceId = H5S.create_simple(1, dims);
//// Create a copy of a standard data type. We will use the
//// resulting H5DataTypeId to create the data set. We could
//// have used the HST.H5Type data directly in the call to
//// H5D.create, but this demonstrates the use of H5T.copy
//// and the use of a H5DataTypeId in H5D.create.
typeId = H5T.copy(H5T.H5Type.NATIVE_INT);
typestring = H5T.copy(H5T.H5Type.C_S1);
H5T.setSize(typestring,6);
//// Set the order to big endian
//H5T.setOrder(typeId, H5T.Order.BE);
//// Set the order to little endian
//H5T.setOrder(typeId, H5T.Order.LE);
//write dset_data into text file (checking)
////string activeDir =
@"D:\HDF\Simulation\HDF5_Simulation_V11_VS2008\PD Data\hasil";
//////create a new subfolder under the current active folder
////if (Directory.Exists(activeDir) == false)
////{
//// System.IO.Directory.CreateDirectory(activeDir);
////}
////int J = 0;
////string newPath;
////string minutes = "";
////int circuit = 1;
////string filename = "hasilarray.txt";
////newPath = System.IO.Path.Combine(activeDir, filename);
////TextWriter tw = new StreamWriter(newPath);
////J = 0;
////for (J = 0; J < numline; J++)
////{
//// tw.WriteLine(dset_data[J].b + "," + dset_data[J].c + "," +
dset_data[J].d + "," + dset_data[J].e);
////}
//////circuit = circuit + 1;
////tw.Close();
////MessageBox.Show("Successfully written to txt file");
try
{
typeId1 =
H5T.create(H5T.CreateClass.COMPOUND,(uint)System.Runtime.InteropServices.Marshal.SizeOf(typeof(struct1)));
H5T.insert(typeId1, "EMT", 0, H5T.H5Type.NATIVE_FLOAT);
H5T.insert(typeId1, "Fractional time", sizeof(float),
H5T.H5Type.NATIVE_INT);
H5T.insert(typeId1, "Charges", sizeof(int) + sizeof(float),
H5T.H5Type.NATIVE_FLOAT);
H5T.insert(typeId1, "Phase angle", sizeof(float) + sizeof(int)
+ sizeof(float), H5T.H5Type.NATIVE_SHORT);
H5T.insert(typeId1, "Minute", sizeof(short) + sizeof(float) +
sizeof(int) + sizeof(float), H5T.H5Type.NATIVE_SHORT);
//H5T.insert(typeId1, "minutes", sizeof(short) + sizeof(float)
+ sizeof(int) + sizeof(float), typestring);
}
catch (Exception et)
{
MessageBox.Show(et.ToString());
}
DateTime range_time_start;
DateTime range_time_end;
range_time_start = DateTime.Now;
try
{
dataSetId = H5D.create(fileId, "/" + circuitid + "/" +
groupname[0] + "/" + groupname[1] + "/" + groupname[2] + "/" + groupname[3],
typeId1, spaceId);
// Write the integer data to the data set.
H5D.write(dataSetId,typeId1 ,new H5Array<struct1>(dset_data));
//write the totaline in date subgroup
int[] totaline = new int[1] { readBackAttr[0] + numline };
H5A.write<int>(attributeId, attributeType, new
H5Array<int>(totaline));
//create attribute type
attributeHourType = H5T.copy(H5T.H5Type.NATIVE_INT);
//create attribute data space
ulong[] attributehourdims = new ulong[1];
attributehourdims[0] = 1;
attributeHourSpace = H5S.create_simple(1, attributehourdims);
//create attribute in the subgroup hour
bool attributehourexist = CheckAttributeInHour(dataSetId,
"numline");
if (attributehourexist == true)
{
//MessageBox.Show("Successfully open attribute");
attributeHourId = H5A.open(dataSetId, "numline");
}
else if (attributehourexist == false)
{
//MessageBox.Show("Successfully create attribute");
attributeHourId = H5A.create(dataSetId, "numline",
attributeHourType, attributeHourSpace);
}
//write the totaline in hour subgroup
int[] totalineinhour = new int[1] { numline };
H5A.write<int>(attributeHourId, attributeHourType, new
H5Array<int>(totalineinhour));
countpd = numline;
H5D.close(dataSetId);
H5T.close(attributeHourType);
H5A.close(attributeHourId);
}
catch (Exception ec)
{
////MessageBox.Show("DataSet already exists: " + ec.ToString());
countpd = 0;
}
totalpd = countpd;
range_time_end = DateTime.Now;
elapsed_time_insert = range_time_end.Subtract(range_time_start);
// Close all the open resources.
H5T.close(attributeType);
H5A.close(attributeId);
H5S.close(spaceId);
H5T.close(typeId1);
H5T.close(typeId);
H5T.close(typestring);
H5G.close(groupcircuitId);
H5G.close(groupyearId);
H5G.close(groupmonthId);
H5G.close(groupdateId);
H5F.close(fileId);
}
private void InsertToHDF5_NoHierarchy(String circuitid, string[] groupname,
struct1[] dset_data, int numline,out int totalpd, out TimeSpan
elapsed_time_insert)
{
int countpd = 0;
bool filexist = CheckFile();
if (filexist == true)
{
fileId = H5F.open(H5File_Name, H5F.OpenMode.ACC_RDWR);
}
//Create a data space to accomodate our 1 dimensional array
//The resulting H5DataSpaceId will be used to create the dataset
ulong[] dims = new ulong[1];
dims[0] = ulong.Parse(numline.ToString());
spaceId = H5S.create_simple(1, dims);
//// Create a copy of a standard data type.
typeId = H5T.copy(H5T.H5Type.NATIVE_INT);
try
{
typeId1 =
H5T.create(H5T.CreateClass.COMPOUND,(uint)System.Runtime.InteropServices.Marshal.SizeOf(typeof(struct1)));
H5T.insert(typeId1, "EMT", 0, H5T.H5Type.NATIVE_FLOAT);
H5T.insert(typeId1, "Fractional time", sizeof(float),
H5T.H5Type.NATIVE_INT);
H5T.insert(typeId1, "Charges", sizeof(int) + sizeof(float),
H5T.H5Type.NATIVE_FLOAT);
H5T.insert(typeId1, "Phase angle", sizeof(float) + sizeof(int)
+ sizeof(float), H5T.H5Type.NATIVE_SHORT);
H5T.insert(typeId1, "Minute", sizeof(short) + sizeof(float) +
sizeof(int) + sizeof(float), H5T.H5Type.NATIVE_SHORT);
//H5T.insert(typeId1, "minutes", sizeof(short) + sizeof(float)
+ sizeof(int) + sizeof(float), typestring);
}
catch (Exception et)
{
MessageBox.Show(et.ToString());
}
DateTime range_time_start;
DateTime range_time_end;
string dsetname = circuitid + "_" + groupname[0] + "_" +
groupname[1] + "_" + groupname[2] + "_" + groupname[3];
//string dsetname = circuitid + "_" + groupname[3];
range_time_start = DateTime.Now;
try
{
dataSetId = H5D.create(fileId,dsetname ,typeId1, spaceId);
// Write the integer data to the data set.
H5D.write(dataSetId,typeId1 ,new H5Array<struct1>(dset_data));
//create attribute type
attributeHourType = H5T.copy(H5T.H5Type.NATIVE_INT);
//create attribute data space
ulong[] attributehourdims = new ulong[1];
attributehourdims[0] = 1;
attributeHourSpace = H5S.create_simple(1, attributehourdims);
//create attribute in the subgroup hour
bool attributehourexist = CheckAttributeInHour(dataSetId,
"numline");
if (attributehourexist == true)
{
//MessageBox.Show("Successfully open attribute");
attributeHourId = H5A.open(dataSetId, "numline");
}
else if (attributehourexist == false)
{
//MessageBox.Show("Successfully create attribute");
attributeHourId = H5A.create(dataSetId, "numline",
attributeHourType, attributeHourSpace);
}
//write the totaline in hour subgroup
int[] totalineinhour = new int[1] { numline };
H5A.write<int>(attributeHourId, attributeHourType, new
H5Array<int>(totalineinhour));
countpd = numline;
H5D.close(dataSetId);
H5T.close(attributeHourType);
H5A.close(attributeHourId);
}
catch (Exception ec)
{
MessageBox.Show(ec.ToString());
countpd = 0;
}
range_time_end = DateTime.Now;
totalpd = countpd;
elapsed_time_insert = range_time_end.Subtract(range_time_start);
// Close all the open resources.
//H5T.close(attributeType);
//H5A.close(attributeId);
H5S.close(spaceId);
H5T.close(typeId1);
//H5T.close(typeId);
//H5T.close(typestring);
//H5G.close(groupcircuitId);
//H5G.close(groupyearId);
//H5G.close(groupmonthId);
//H5G.close(groupdateId);
H5F.close(fileId);
}
-------------------------------------------------------------------------------------------------------------------
Retrieve HDF_WithHierarchy
foreach (string d in daterange)
{
string[] groupname = new string[4];
ReturnArrayinMyLaptop(d, out groupname);
// check if that group is already exist
circuitid = "SystemIdent-1" + cmbCircuitID2.SelectedItem;
//MessageBox.Show("circuit:" + circuitid);
bool circuitexist = CheckGroup(circuitid);
if (circuitexist == true)
{
//MessageBox.Show("Successfully open circuit group");
groupcircuitId = H5G.open(fileId, "/" + circuitid);
//create year subgroup but check it first
bool yearexist = CheckGroup(circuitid + "/" + groupname[0]);
if (yearexist == true)
{
//MessageBox.Show("Successfully open year subgroup");
groupyearId = H5G.open(fileId, "/" + circuitid + "/" +
groupname[0]);
//create month subgroup but check it first
bool monthexist = CheckGroup(circuitid + "/" +
groupname[0] + "/" + groupname[1]);
if (monthexist == true)
{
//MessageBox.Show("Successfully open month
subgroup");
groupmonthId = H5G.open(fileId, "/" + circuitid +
"/" + groupname[0] + "/" + groupname[1]);
//open date subgroup but check it first
bool datexist = CheckGroup(circuitid + "/" +
groupname[0] + "/" + groupname[1] + "/" + groupname[2]);
if (datexist == true)
{
//MessageBox.Show("Successfully open date
subgroup");
groupdateId = H5G.open(fileId, "/" + circuitid +
"/" + groupname[0] + "/" + groupname[1] + "/" + groupname[2]);
//loop for every hour in one day
int ij = 0;
while (ij <= 23)
{
try
{
dataSetId = H5D.open(fileId, "/" +
circuitid + "/" + groupname[0] + "/" + groupname[1] + "/" + groupname[2] + "/"
+ ij + ":00:00");
bool attributehourexist =
CheckAttributeInHour(dataSetId, "numline");
if (attributehourexist == true)
{
//MessageBox.Show("Successfully open
attribute");
attributeHourId =
H5A.open(dataSetId, "numline");
//create attribute type
attributeHourType =
H5T.copy(H5T.H5Type.NATIVE_INT);
int[] readBackAttr = new int[1];
H5A.read<int>(attributeHourId,
attributeHourType, new H5Array<int>(readBackAttr));
int indexdataset = 0;
struct1[] readDataBack = new
struct1[readBackAttr[0]];
// Read the integer data back from
the data set
H5D.read(dataSetId,typeId1, new
H5Array<struct1>(readDataBack));
//move data from temporary array to
permanent array
indexarray = startindex;
while (indexarray < startindex +
readBackAttr[0])
{
FullArray[indexarray].a =
readDataBack[indexdataset].a;
FullArray[indexarray].b =
readDataBack[indexdataset].b;
FullArray[indexarray].c =
readDataBack[indexdataset].c;
FullArray[indexarray].d =
readDataBack[indexdataset].d;
FullArray[indexarray].e =
readDataBack[indexdataset].e;
indexdataset = indexdataset + 1;
indexarray = indexarray + 1;
}
//to get the start index for the
array for the next dataset
startindex = startindex +
readBackAttr[0];
H5T.close(attributeHourType);
H5A.close(attributeHourId);
}
else if (attributehourexist == false)
{
//MessageBox.Show("Attribute is not
exist");
}
H5D.close(dataSetId);
ij = ij + 1;
}
catch (Exception ec)
{
//MessageBox.Show(ec.ToString());
ij = ij + 1;
}
}
H5G.close(groupdateId);
}
else if (datexist == false)
{
//MessageBox.Show("Date does not exist");
}
//H5G.close(groupmonthId);
}
else if (monthexist == false)
{
//MessageBox.Show("Month does not exist");
}
//H5G.close(groupyearId);
}
else if (yearexist == false)
{
//MessageBox.Show("Year does not exist");
}
H5G.close(groupcircuitId);
}
else if (circuitexist == false)
{
//MessageBox.Show("Circuit does not exist");
}
// Close all the open resources.
//H5T.close(typeId);
//H5S.close(spaceId);
}
---------------------------------------------------------------------------------------------------------------------
Retrieve HDF_NoHierarchy
foreach (string d in daterange)
{
string[] groupname = new string[4];
ReturnArrayinMyLaptop(d, out groupname);
// check if that group is already exist
circuitid = "SystemIdent-" + cmbCircuitID2.SelectedItem;
//MessageBox.Show("circuit:" + circuitid);
//loop for every hour in one day
int ij = 0;
while (ij <= 23)
{
try
{
dataSetId = H5D.open(fileId, circuitid + "_" +
groupname[0] + "_" + groupname[1] + "_" + groupname[2] + "_" + ij + ":00:00");
bool attributehourexist =
CheckAttributeInHour(dataSetId, "numline");
if (attributehourexist == true)
{
//MessageBox.Show("Successfully open attribute");
attributeHourId = H5A.open(dataSetId, "numline");
//create attribute type
attributeHourType = H5T.copy(H5T.H5Type.NATIVE_INT);
int[] readBackAttr = new int[1];
H5A.read<int>(attributeHourId, attributeHourType,
new H5Array<int>(readBackAttr));
int indexdataset = 0;
struct1[] readDataBack = new
struct1[readBackAttr[0]];
// Read the integer data back from the data set
H5D.read(dataSetId, typeId1, new
H5Array<struct1>(readDataBack));
//move data from temporary array to permanent array
indexarray = startindex;
while (indexarray < startindex + readBackAttr[0])
{
FullArray[indexarray].a =
readDataBack[indexdataset].a;
FullArray[indexarray].b =
readDataBack[indexdataset].b;
FullArray[indexarray].c =
readDataBack[indexdataset].c;
FullArray[indexarray].d =
readDataBack[indexdataset].d;
FullArray[indexarray].e =
readDataBack[indexdataset].e;
indexdataset = indexdataset + 1;
indexarray = indexarray + 1;
}
//to get the start index for the array for the next
dataset
startindex = startindex + readBackAttr[0];
H5T.close(attributeHourType);
H5A.close(attributeHourId);
}
else if (attributehourexist == false)
{
//MessageBox.Show("Attribute is not exist");
}
H5D.close(dataSetId);
ij = ij + 1;
}
catch (Exception ec)
{
//MessageBox.Show(ec.ToString());
ij = ij + 1;
}
}
}
_______________________________________________
Hdf-forum is for HDF software users discussion.
[email protected]
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
