H5hut/src_old
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated h5_lustre.c to user new macros; changed library targets to ../lib/lib*.a to avoid the mv and mkdir hacks in Makefile.am; fixed problem in h5u_test with H5PartSetViewIndices; OPEN BUG with h5u_test for >1 MPI task (strange pthread setfault?)

Browse Source
This commit is contained in:
Marc Howison
2011-05-05 17:58:05 +00:00
parent c676222a03
commit 11fd3c10c5
22 changed files with 122 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/C/H5.c
+12 -2
View File
@@ -41,6 +41,16 @@
Open file with name \c filbename. This function is available in the parallel
and serial version. In the serial case \c comm may have any value.
File mode flags are:
- H5_O_RDONLY: only reading allowed
- H5_O_WRONLY: create new file, dataset must not exist
- H5_O_APPEND: allows to append a new datasets to an existing file
- H5_O_RDWR: dataset may exist
You can also select a "virtual file driver" in the HDF5 layer using:
- H5_VFD_INDEPENDENT: MPI-IO in independent (asynchronous) mode
- H5_VFD_MPIPOSIX: parallel I/O implemented directly by HDF5, bypassing MPI-IO
\return File handle.
\return NULL on error.
*/
@@ -59,7 +69,7 @@ H5OpenFile (
/*!
\ingroup h5hut_file
Close file.
Close file and free all memory associated with the file handle.
\return \c H5_SUCCESS or error code
*/
@@ -74,7 +84,7 @@ H5CloseFile (
/*!
\ingroup h5hut_file
Close file.
Verify that the file handle points to a valid H5hut file structure.
\return \c H5_SUCCESS or error code
*/
src/C/H5Part.c
+6 -1
View File
@@ -539,6 +539,11 @@ H5PartSetView (
is set, or the number of particles in a dataset changes, or the view is
"unset" by calling \c H5PartSetViewIndices(NULL,0);
When you perform a read or write on a view consisting of indices, it
is assumed that your buffer is \b unpacked, meaning that there is room
for all the intermediate values (which will not be touched by the read
or write).
Before you set a view, the \c H5PartGetNumParticles() will return the
total number of particles in the current time-step (even for the parallel
reads). However, after you set a view, it will return the number of
@@ -549,7 +554,7 @@ H5PartSetView (
h5_err_t
H5PartSetViewIndices (
h5_file_t *f, /*!< [in] Handle to open file */
const h5_id_t *indices, /*!< [in] List of indices */
const h5_size_t *indices, /*!< [in] List of indices */
h5_size_t nelems /*!< [in] Size of list */
) {
H5_API_ENTER3 (h5_err_t,
src/C/Makefile.am
+3 -6
View File
@@ -9,7 +9,7 @@ INCLUDES = -I../include @INCLUDES@
lib_LIBRARIES = @LIB_C@
# Listing of all possible targets that I may build.
EXTRA_LIBRARIES = libH5hutC.a
EXTRA_LIBRARIES = ../lib/libH5hutC.a
# Header files that I wish to install in $(prefix)/include
include_HEADERS = \
@@ -31,7 +31,7 @@ include_HEADERS = \
EXTRA_HEADERS =
# Listing of sources
libH5hutC_a_SOURCES = \
___lib_libH5hutC_a_SOURCES = \
H5.c \
H5_attribs.c \
H5_attachments.c \
@@ -45,10 +45,7 @@ libH5hutC_a_SOURCES = \
H5Fed_store.c \
H5Fed_tags.c
all: all-am install-libs-local
install-libs-local:
-cp ${lib_LIBRARIES} ../lib
all: all-am
clean: clean-am
src/Fortran/H5Part_F.c
+1 -1
View File
@@ -146,7 +146,7 @@ h5pt_setview (
h5_file_t *filehandle = (h5_file_t*)(size_t)*f;
h5_set_funcname( filehandle, __func__ );
return h5u_set_view ( filehandle, *start, *end );
return h5u_set_view ( filehandle, (*start)-1, (*end)-1 );
}
h5_err_t
src/Fortran/H5_F.c
+6 -6
View File
@@ -78,7 +78,7 @@ h5_openr (
) {
char *file_name2 = h5_strdupfor2c ( file_name, l_file_name );
h5_file_t* f = h5_open_file ( file_name2, H5_O_RDONLY, 0, __func__ );
h5_file_t* f = h5_open_file ( file_name2, H5_O_RDONLY, 0 );
free ( file_name2 );
return (h5_int64_t)(size_t)f;
}
@@ -90,7 +90,7 @@ h5_openw (
) {
char *file_name2 = h5_strdupfor2c ( file_name, l_file_name );
h5_file_t* f = h5_open_file ( file_name2, H5_O_WRONLY, 0, __func__ );
h5_file_t* f = h5_open_file ( file_name2, H5_O_WRONLY, 0 );
free ( file_name2 );
return (h5_int64_t)(size_t)f;
}
@@ -102,7 +102,7 @@ h5pt_opena (
) {
char *file_name2 = h5_strdupfor2c ( file_name, l_file_name );
h5_file_t* f = h5_open_file ( file_name2, H5_O_APPEND, 0, __func__ );
h5_file_t* f = h5_open_file ( file_name2, H5_O_APPEND, 0 );
free ( file_name2 );
return (h5_int64_t)(size_t)f;
}
@@ -123,7 +123,7 @@ h5_openr_par (
h5_int32_t fbits = H5_O_RDONLY | _flagsfor2c ( flags2 );
h5_file_t* f = h5_open_file ( file_name2, fbits, ccomm, __func__ );
h5_file_t* f = h5_open_file ( file_name2, fbits, ccomm );
free ( file_name2 );
free ( flags2 );
@@ -145,7 +145,7 @@ h5_openw_par (
h5_int32_t fbits = H5_O_WRONLY | _flagsfor2c ( flags2 );
h5_file_t* f = h5_open_file ( file_name2, ccomm, fbits, __func__ );
h5_file_t* f = h5_open_file ( file_name2, ccomm, fbits );
free ( file_name2 );
free ( flags2 );
@@ -168,7 +168,7 @@ h5pt_opena_par_align (
h5_int32_t fbits = H5_O_APPEND | _flagsfor2c ( flags2 );
h5_file_t* f = h5_open_file( file_name2, ccomm, fbits, __func__ );
h5_file_t* f = h5_open_file( file_name2, ccomm, fbits );
free ( file_name2 );
free ( flags2 );
src/Fortran/Makefile.am
+8 -7
View File
@@ -26,15 +26,16 @@ nodist_include_HEADERS = ../include/H5hutF.h
lib_LIBRARIES = @LIB_FORTRAN@
# Listing of all possible targets that I may build.
EXTRA_LIBRARIES = libH5hutF.a
EXTRA_LIBRARIES = ../lib/libH5hutF.a
libH5hutF_a_SOURCES = H5_F.c \
H5_attribs_F.c \
H5Part_F.c \
H5Block_F.c \
H5Block_readwrite_F.c
___lib_libH5hutF_a_SOURCES = \
H5_F.c \
H5_attribs_F.c \
H5Part_F.c \
H5Block_F.c \
H5Block_readwrite_F.c
libH5hutF_a_DEPENDENCIES = ../include/H5hutF.h
___lib_libH5hutF_a_DEPENDENCIES = ../include/H5hutF.h
../include/H5hutF.h: $(F90_FILES)
awk '/INTEGER\*8 FUNCTION/{print "\t" $$1 " " $$3}' $^ >$@
src/h5core/Makefile.am
+11 -12
View File
@@ -32,19 +32,22 @@ EXTRA_HEADERS = \
../include/h5core/h5t_retrieve.h \
../include/h5core/h5t_storemesh.h \
../include/h5core/h5t_tags.h \
h5b_errorhandling_private.h \
h5b_model_private.h \
h5b_types_private.h \
h5_attribs_private.h \
h5_core_private.h \
h5_errorhandling_private.h \
h5_fcmp_private.h \
h5_hdf5_private.h \
h5_hsearch_private.h \
h5_lustre_private.h \
h5_maps_private.h \
h5_mpi_private.h \
h5_qsort_private.h \
h5_readwrite_private.h \
h5_syscall_private.h \
h5_types_private.h \
h5b_errorhandling_private.h \
h5b_model_private.h \
h5b_types_private.h \
h5t_access_private.h \
h5t_adjacencies_private.h \
h5t_consts_private.h \
@@ -60,7 +63,6 @@ EXTRA_HEADERS = \
h5t_store_trim_private.h \
h5t_tags_private.h \
h5t_types_private.h \
h5_types_private.h \
h5u_errorhandling_private.h \
h5u_types_private.h
@@ -68,13 +70,13 @@ EXTRA_HEADERS = \
EXTRA_DIST = $(EXTRA_HEADERS)
# What to build... Will be determined by configure script.
lib_LIBRARIES = libH5hut.a
lib_LIBRARIES = ../lib/libH5hut.a
# Listing of all possible targets that I may build.
EXTRA_LIBRARIES = libH5hut.a
EXTRA_LIBRARIES = ../lib/libH5hut.a
# Listing of sources
libH5hut_a_SOURCES = \
___lib_libH5hut_a_SOURCES = \
h5_attach.c \
h5_attribs.c \
h5_errorhandling.c \
@@ -117,12 +119,9 @@ libH5hut_a_SOURCES = \
h5t_store_trim.c \
h5t_tags.c
libH5hut_a_DEPENDENCIES = $(EXTRA_HEADERS)
___lib_libH5hut_a_DEPENDENCIES = $(EXTRA_HEADERS)
all: all-am install-libs-local
install-libs-local:
-cp ${lib_LIBRARIES} ../lib
all: all-am
install-exec-local:
@$(INSTALL) -d $(DESTDIR)$(includedir)/h5core
src/h5core/h5_errorhandling.c
+1 -1
View File
@@ -83,7 +83,7 @@ h5_set_debuglevel (
\return current debug level
*/
h5_id_t
h5_err_t
h5_get_debuglevel (
void
) {
src/h5core/h5_hdf5.c
+2 -2
View File
@@ -262,7 +262,7 @@ hdf5_close_dataset (
"dataset_id=%d (%s)",
dataset_id,
hdf5_get_objname (dataset_id));
if (dataset_id == 0 || dataset_id == -1)
if (dataset_id < 0)
HDF5_WRAPPER_LEAVE (H5_SUCCESS);
if (H5Dclose (dataset_id) < 0) {
@@ -537,7 +537,7 @@ hdf5_select_elements_of_dataspace (
herr = H5Sselect_elements (
space_id,
op,
nelems,
(size_t)nelems,
indices);
} else {
herr = H5Sselect_none ( space_id );
src/h5core/h5_lustre.c
+30 -24
View File
@@ -8,6 +8,7 @@
#define __USE_GNU
#include <fcntl.h>
#undef __USE_GNU
#include <unistd.h>
#include <lustre/liblustreapi.h>
#include "h5core/h5_core.h"
@@ -32,25 +33,29 @@ _get_lustre_stripe_size(h5_file_t *const f, const char *path )
{
size_t nbytes = sizeof(struct lov_user_md) +
INIT_ALLOC_NUM_OSTS * sizeof(struct lov_user_ost_data);
struct lov_user_md *lum;
TRY( lum = h5_calloc (1, nbytes) );
struct lov_user_md *lum = h5_calloc(1, nbytes);
if (!lum) {
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"cannot allocate lustre struct");
return -1;
}
lum->lmm_magic = LOV_USER_MAGIC;
int fd = open64(path, O_RDONLY);
int fd = open(path, O_RDONLY);
if (fd < 0) {
extern int errno;
if (errno == EINVAL)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: a flag is invalid!");
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"open: a flag is invalid!");
else if (errno == EACCES)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: access denied or file does not exist!");
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"open: access denied or file does not exist!");
else if (errno == ENAMETOOLONG)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: path is too long!");
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"open: path is too long!");
else
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: unspecific error!");
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"open: unspecific error!");
return -1;
}
@@ -58,22 +63,22 @@ _get_lustre_stripe_size(h5_file_t *const f, const char *path )
if (ret == -1) {
extern int errno;
if (errno == EBADF)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"ioctl: bad file handle!");
else if (errno == EINVAL)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"ioctl: invalid argument!");
else if (errno == EIO)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"ioctl: physical I/O problem!");
else if (errno == ENOTTY)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"ioctl: file handle does not accept control functions!");
else if (errno == ENODEV)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"ioctl: driver doesn't support control functions!");
else
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
h5_error(H5_ERR_INTERNAL, MSG_HEADER
"ioctl: unspecific error!");
return -1;
}
@@ -95,6 +100,7 @@ h5_optimize_for_lustre (
const char *filename
) {
H5_CORE_API_ENTER2 (h5_err_t, "f=0x%p, filename=\"%s\"", f, filename);
ssize_t stripe_size;
if ( f->myproc == 0 )
{
@@ -119,23 +125,23 @@ h5_optimize_for_lustre (
h5_free(path);
}
TRY( h5priv_mpi_bcast(f, &stripe_size, 1, MPI_LONG_LONG, 0, f->comm) );
h5_info(f, MSG_HEADER
TRY( h5priv_mpi_bcast(&stripe_size, 1, MPI_LONG_LONG, 0, f->comm) );
h5_info(MSG_HEADER
"Found lustre stripe size of %lld bytes",
(long long)stripe_size);
hsize_t btree_ik = (stripe_size - 4096) / 96;
hsize_t btree_bytes = 64 + 96*btree_ik;
h5_info(f, MSG_HEADER
h5_info(MSG_HEADER
"Setting HDF5 btree ik to %lld (= %lld bytes at rank 3)",
(long long)btree_ik, (long long)btree_bytes);
TRY( hdf5_set_btree_ik_property(f, f->create_prop, btree_ik) );
TRY( hdf5_set_btree_ik_property(f->create_prop, btree_ik) );
/* set alignment to lustre stripe size */
TRY( hdf5_set_alignment_property(f,
TRY( hdf5_set_alignment_property(
f->access_prop, 0, stripe_size) );
h5_info(f, MSG_HEADER "Disabling metadata cache flushes.");
h5_info(MSG_HEADER "Disabling metadata cache flushes.");
/* disable metadata cache flushes */
/* defer metadata writes */
H5AC_cache_config_t config;
@@ -149,7 +155,7 @@ h5_optimize_for_lustre (
config.flash_incr_mode = H5C_flash_incr__off;
TRY( H5Pset_mdc_config( f->access_prop, &config ) );
return H5_SUCCESS;
H5_CORE_API_RETURN (H5_SUCCESS);
}
#endif // H5_USE_LUSTRE
src/h5core/h5_openclose.c
+7 -5
View File
@@ -348,7 +348,8 @@ h5_set_stepname_fmt (
strncpy (
f->prefix_step_name,
name,
H5_STEPNAME_LEN - 1);
//H5_STEPNAME_LEN - 1);
63);
f->width_step_idx = width;
H5_CORE_API_RETURN (H5_SUCCESS);
@@ -474,10 +475,11 @@ h5_strdupfor2c (
char *dup = (char*)malloc ( len + 1 );
strncpy ( dup, s, len );
char *p = dup + len;
do {
*p-- = '\0';
} while ( *p == ' ' );
dup[len] = '\0';
for (int i=len-1; i>=0; i--) {
if (dup[i] == ' ') dup[i] = '\0';
else break;
}
return dup;
}
src/h5core/h5_readwrite.c
+1 -1
View File
@@ -269,7 +269,7 @@ h5_normalize_dataset_name (
H5_CORE_API_ENTER2 (h5_err_t,
"name=\"%s\", name2=\"%s\"",
name, name2);
if ( strlen(name) > H5_DATANAME_LEN ) {
if ( strlen(name) > H5_DATANAME_LEN-1 ) {
strncpy ( name2, name, H5_DATANAME_LEN-1 );
name2[H5_DATANAME_LEN-1] = '\0';
h5_warn ("Truncated name '%s' to '%s'.", name, name2);
src/h5core/h5u_model.c
+8 -8
View File
@@ -275,8 +275,8 @@ h5u_set_view (
h5_err_t
h5u_set_view_indices (
h5_file_t *const f, /*!< [in] Handle to open file */
const h5_id_t *const indices, /*!< [in] List of indices */
const h5_size_t nelems /*!< [in] Size of list */
const h5_size_t *const indices, /*!< [in] List of indices */
h5_size_t nelems /*!< [in] Size of list */
) {
H5_CORE_API_ENTER3 (h5_err_t,
"f=0x%p, indices=0x%p, nelems=%llu",
@@ -299,7 +299,7 @@ h5u_set_view_indices (
For now, we interpret start=-1 to mean 0 and
end==-1 to mean end of file
*/
TRY ( total = (hsize_t) h5u_get_num_particles ( f ) );
TRY ( total = h5u_get_num_particles ( f ) );
if ( total == 0 ) {
/* No datasets have been created yet and no veiws are set.
* We have to leave the view empty because we don't know how
@@ -309,19 +309,19 @@ h5u_set_view_indices (
if ( total == 0 ) return H5_SUCCESS;
u->nparticles = (hsize_t) nelems;
u->nparticles = nelems;
h5_debug ("This view selected %lld particles.", (long long)u->nparticles );
/* declare overall data size but then will select a subset */
TRY (u->diskshape = hdf5_create_dataspace (1, &total, NULL));
/* declare local memory datasize */
total = (size_t)u->nparticles;
total = u->nparticles;
TRY (u->memshape = hdf5_create_dataspace (1, &total, &dmax));
TRY (hdf5_select_elements_of_dataspace (
u->diskshape,
H5S_SELECT_SET,
nelems, (hsize_t*)indices ) );
u->diskshape,
H5S_SELECT_SET,
nelems, indices ) );
u->viewindexed = 1;
src/include/H5Part.h
+1 -1
View File
@@ -129,7 +129,7 @@ H5PartSetView (
h5_err_t
H5PartSetViewIndices (
h5_file_t *f, /*!< [in] Handle to open file */
const h5_id_t *indices, /*!< [in] List of indices */
const h5_size_t *indices, /*!< [in] List of indices */
h5_size_t nelems /*!< [in] Size of list */
);
src/include/h5core/h5u_model.h
+2 -2
View File
@@ -33,8 +33,8 @@ h5u_set_view (
h5_err_t
h5u_set_view_indices (
h5_file_t *const f,
const h5_id_t *const indices,
const h5_size_t nelems
const h5_size_t *const indices,
h5_size_t nelems
);
h5_err_t