Hi,
In 2012 we wrote and tested our functions to use MPI I/O to have good
performances while doing I/O on a Lustre filesystem. Everything was fine
about "striping_factor" we passed to file creation.
Now I am trying to verify some performance degradation we observed and I
am surprised because it looks like I am unable to create a new file with
a given "striping_factor" with any mpi flavor.
I attached a simple example for file creation with hints, and tried it
the following way with OpenMPI:
OpenMPI-4.0.3:
which mpicc
/cvmfs/soft.computecanada.ca/easybuild/software/2020/avx2/Compiler/gcc9/openmpi/4.0.3/bin/mpicc
mpicc -o s simple_file_create_with_hint.c
rm -f foo && mpiexec -n 1 --mca io romio321 ./s foo && lfs getstripe foo
Creating the file by MPI_file_open : foo
Informations on file:
Key is 'striping_factor' and worth: '2'
...closing the file foo
Informations on file:
foo
lmm_stripe_count: 1
lmm_stripe_size: 1048576
lmm_pattern: raid0
lmm_layout_gen: 0
lmm_stripe_offset: 49
obdidx objid objid group
49 485863591 0x1cf5b0a7 0
#Not forcing romio321:
rm -f foo && mpiexec -n 1 ./s foo && lfs getstripe foo
Creating the file by MPI_file_open : foo
Informations on file:
Key is 'striping_factor' and worth: '2'
...closing the file foo
Informations on file:
foo
lmm_stripe_count: 1
lmm_stripe_size: 1048576
lmm_pattern: raid0
lmm_layout_gen: 0
lmm_stripe_offset: 19
obdidx objid objid group
19 482813449 0x1cc72609 0
First, as you can see, even if I ask for a striping_factor of 2, I only
get one! I tried to write some data too, but it changed nothing...
Where am I wrong?
Second, I was expecting that when I re-open the file for read-only, I
would have some information in "MPI_Info" but it is empty... is that normal?
For example, using mpich-3.2.1 I have the following output:
which mpicc
/cvmfs/soft.computecanada.ca/easybuild/software/2017/avx2/Compiler/gcc7.3/mpich/3.2.1/bin/mpicc
mpicc -o s simple_file_create_with_hint.c
rm -f foo && mpiexec -n 1 ./s foo && lfs getstripe foo
Creating the file by MPI_file_open : foo
Informations on file:
Key is 'cb_buffer_size' and worth: '16777216'
Key is 'romio_cb_read' and worth: 'automatic'
Key is 'romio_cb_write' and worth: 'automatic'
Key is 'cb_nodes' and worth: '1'
Key is 'romio_no_indep_rw' and worth: 'false'
Key is 'romio_cb_pfr' and worth: 'disable'
Key is 'romio_cb_fr_types' and worth: 'aar'
Key is 'romio_cb_fr_alignment' and worth: '1'
Key is 'romio_cb_ds_threshold' and worth: '0'
Key is 'romio_cb_alltoall' and worth: 'automatic'
Key is 'ind_rd_buffer_size' and worth: '4194304'
Key is 'ind_wr_buffer_size' and worth: '524288'
Key is 'romio_ds_read' and worth: 'automatic'
Key is 'romio_ds_write' and worth: 'automatic'
Key is 'cb_config_list' and worth: '*:1'
Key is 'romio_filesystem_type' and worth: 'UFS: Generic ROMIO driver for
all UNIX-like file systems'
Key is 'romio_aggregator_list' and worth: '0 '
...closing the file foo
Informations on file:
Key is 'cb_buffer_size' and worth: '16777216'
Key is 'romio_cb_read' and worth: 'automatic'
Key is 'romio_cb_write' and worth: 'automatic'
Key is 'cb_nodes' and worth: '1'
Key is 'romio_no_indep_rw' and worth: 'false'
Key is 'romio_cb_pfr' and worth: 'disable'
Key is 'romio_cb_fr_types' and worth: 'aar'
Key is 'romio_cb_fr_alignment' and worth: '1'
Key is 'romio_cb_ds_threshold' and worth: '0'
Key is 'romio_cb_alltoall' and worth: 'automatic'
Key is 'ind_rd_buffer_size' and worth: '4194304'
Key is 'ind_wr_buffer_size' and worth: '524288'
Key is 'romio_ds_read' and worth: 'automatic'
Key is 'romio_ds_write' and worth: 'automatic'
Key is 'cb_config_list' and worth: '*:1'
Key is 'romio_filesystem_type' and worth: 'UFS: Generic ROMIO driver for
all UNIX-like file systems'
Key is 'romio_aggregator_list' and worth: '0 '
foo
lmm_stripe_count: 1
lmm_stripe_size: 1048576
lmm_pattern: raid0
lmm_layout_gen: 0
lmm_stripe_offset: 67
obdidx objid objid group
67 357367195 0x154cfd9b 0
but still have only a striping_factor of 1 on the created file...
Thanks,
Eric
--
Eric Chamberland, ing., M. Ing
Professionnel de recherche
GIREF/Université Laval
(418) 656-2131 poste 41 22 42
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
/*
* Simple function to abort execution and print an error based on MPI return values:
*/
void abortOnError(int ierr) {
if (ierr != MPI_SUCCESS) {
printf("ERROR Returned by MPI: %d\n",ierr);
char* lCharPtr = (char*) malloc(sizeof(char)*MPI_MAX_ERROR_STRING);
int lLongueur = 0;
MPI_Error_string(ierr,lCharPtr, &lLongueur);
printf("ERROR_string Returned by MPI: %s\n",lCharPtr);
free(lCharPtr);
MPI_Abort( MPI_COMM_WORLD, ierr );
}
}
/*
* Here I use only the first hint for now, but you can try a few to see the difference:
*/
const int lNbHints = 1;
char *lHints [] = {"striping_factor",
"striping_unit",
"access_style"};
char *lHintsValues[] = {"2",
"1048576",
"write_once,random"};
/*
* Simple function to extract all information in MPI_Info:
*/
void displayFileInfo(MPI_File pFile)
{
printf( "Informations on file:\n");fflush(stdout);
char lKey[MPI_MAX_INFO_KEY];
char lValue[MPI_MAX_INFO_VAL];
int lFlag = 0;
MPI_Info lInfo;
int lNKeys = 0;
abortOnError(MPI_File_get_info( pFile, &lInfo ));
abortOnError(MPI_Info_get_nkeys( lInfo, &lNKeys ));
/* Note that an implementation is allowed to ignore the set_info, so we'll accept either the original or the updated version */
for (int i = 0 ; i < lNKeys; ++i ) {
abortOnError(MPI_Info_get_nthkey( lInfo, i, lKey ));
abortOnError(MPI_Info_get ( lInfo, lKey, MPI_MAX_INFO_VAL, lValue, &lFlag ));
if (lFlag) {
printf( "Key is '%s' and worth: '%s'\n", lKey, lValue);fflush(stdout);
}
else {
printf( "Error getting key is '%s' \n", lKey);fflush(stdout);
}
}
MPI_Info_free( &lInfo );
}
/*
* Function to create a file with *hints*:
*/
void createFileCollectivelyWithHint(char* pFileName)
{
int size, rank;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Comm_size( MPI_COMM_WORLD, &size );
MPI_File lFile = 0;
MPI_Info lInfoIn;
abortOnError(MPI_Info_create( &lInfoIn ));
for (int i = 0 ; i < lNbHints; ++i ) {
abortOnError(MPI_Info_set( lInfoIn, lHints[i], lHintsValues[i] ));
}
printf("Creating the file by MPI_file_open : %s\n", pFileName);
abortOnError(MPI_File_open( MPI_COMM_WORLD, pFileName, ( MPI_MODE_CREATE | MPI_MODE_WRONLY ), lInfoIn, &lFile ));
displayFileInfo(lFile);
if (lFile) {
printf(" ...closing the file %s\n", pFileName);
abortOnError(MPI_File_close(&lFile ));
}
}
int main(int argc, char *argv[])
{
MPI_Init(&argc, &argv);
if (2 != argc) {
printf("ERROR: you must specify a filename to create.\n");
MPI_Finalize();
return 1;
}
char* lFileName = argv[1];
createFileCollectivelyWithHint(lFileName);
/* Re-open the file in read-only mode just to see MPI_Info values */
MPI_Info lInfo = MPI_INFO_NULL;
MPI_File lFile = 0;
abortOnError(MPI_Info_create( &lInfo ));
abortOnError(MPI_File_open( MPI_COMM_WORLD, lFileName, ( MPI_MODE_RDONLY ), lInfo, &lFile ));
displayFileInfo(lFile);
abortOnError(MPI_File_close(&lFile ));
MPI_Finalize();
return 0;
}
