On Sun, 12 Aug 2007, Lingling Tang wrote:
> I am using meep-mpi (LAM), and find that if I use multiple CPUs,
> output-Xfield does not work. The computers keep running, but will never
> finish the first .h5 file output. Instead, to output each component of
> the field (output-Xfield-x ...) is fine. If I use just single CPU, the
> output-Xfield works.
>
> This doesn't affect our simulation, but I am serious why this happens.
> Is this normal, or my computer setting is incorrect?
This is a bug.
What's happening is that you are linking to the non-MPI version of HDF5.
For the MPI version, HDF5 handles all of the file synchronization itself,
but in the non-MPI HDF5 Meep has some hacks to allow each process to write
to the file in sequence (which works okay on a few processors).
Unfortunately, you've found a bug in Meep's synchronization for the
non-MPI HDF5, which causes it to deadlock when writing multiple field
components to the same file.
There are at lesat three workarounds:
1) Write individual components to individual files, as you've discovered.
2) Use a version of HDF5 that is compiled with MPI support. (This is a
good idea anyway, especially if you have lots of processors.)
3) Apply the patch below, which fixes the problem (at least, it works
for me), and will be in the next Meep release.
Regards,
Steven G. Johnson
------------------------------------------------------------
diff -rN -u old-meep/libctl/meep.scm.in new-meep/libctl/meep.scm.in
--- old-meep/libctl/meep.scm.in 2007-08-15 15:41:52.288757463 -0400
+++ new-meep/libctl/meep.scm.in 2007-08-15 15:41:52.344749246 -0400
@@ -591,7 +591,11 @@
(list (meep-fields-open-h5file fields fname (meep-h5file-WRITE)
(get-filename-prefix) true))
'())))
- (map (lambda (c) (apply output-component (cons c f))) cs)
+ (map (lambda (c)
+ (apply output-component (cons c f))
+ (if (null? output-append-h5)
+ (meep-h5file-prevent-deadlock (car f))))
+ cs)
(if (null? output-append-h5) (delete-meep-h5file (car f))))
(if (null? output-append-h5)
(output-h5-hook (meep-fields-h5file-name
diff -rN -u old-meep/src/h5file.cpp new-meep/src/h5file.cpp
--- old-meep/src/h5file.cpp 2007-08-15 15:41:52.310754235 -0400
+++ new-meep/src/h5file.cpp 2007-08-15 15:41:52.374744845 -0400
@@ -66,7 +66,7 @@
# endif
#endif
-#ifdef HAVE_H5PSET_FAPL_MPIO
+#if defined(HAVE_H5PSET_FAPL_MPIO) || !defined(HAVE_MPI)
# define IF_EXCLUSIVE(yes,no) no
#else
# define IF_EXCLUSIVE(yes,no) yes
@@ -113,12 +113,14 @@
#ifdef HAVE_HDF5
hid_t access_props = H5Pcreate (H5P_FILE_ACCESS);
-# if defined(HAVE_MPI) && defined(HAVE_H5PSET_FAPL_MPIO)
+# ifdef HAVE_MPI
+# ifdef HAVE_H5PSET_FAPL_MPIO
if (parallel)
H5Pset_fapl_mpio(access_props, MPI_COMM_WORLD, MPI_INFO_NULL);
-# else
+# else
if (parallel)
begin_critical_section(h5io_critical_section_tag);
+# endif
# endif
if (mode != WRITE || IF_EXCLUSIVE(parallel && !am_master(), 0))
@@ -143,13 +145,14 @@
void h5file::close_id() {
unset_cur();
#ifdef HAVE_HDF5
- if (HID(id) >= 0)
+ if (HID(id) >= 0) {
H5Fclose(HID(id));
- HID(id) = -1;
+ IF_EXCLUSIVE(if (parallel)
+ end_critical_section(h5io_critical_section_tag++),
+ (void) 0);
+ }
#endif
- IF_EXCLUSIVE(if (parallel)
- end_critical_section(h5io_critical_section_tag++),
- (void) 0);
+ HID(id) = -1;
if (mode == WRITE) mode = READWRITE; // don't re-create on re-open
}
_______________________________________________
meep-discuss mailing list
[email protected]
http://ab-initio.mit.edu/cgi-bin/mailman/listinfo/meep-discuss
