diff --git a/.gitattributes b/.gitattributes index 44e5224..dab20d7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -376,7 +376,6 @@ src/C/H5Fed_store.c -text src/C/H5Fed_tags.c -text src/C/H5Part.c -text src/C/H5_attribs.c -text -src/C/H5_inquiry.c -text src/C/Makefile.am -text src/C/generate-h5b-readwrite.py -text src/Fortran/H5.f90 -text @@ -404,6 +403,8 @@ src/h5core/h5_hdf5.c -text src/h5core/h5_hdf5_private.h -text src/h5core/h5_hsearch.c -text src/h5core/h5_hsearch_private.h -text +src/h5core/h5_lustre.c -text +src/h5core/h5_lustre_private.h -text src/h5core/h5_maps.c -text src/h5core/h5_model.c -text src/h5core/h5_mpi.c -text @@ -473,7 +474,6 @@ src/include/H5Fed_store.h -text src/include/H5Fed_tags.h -text src/include/H5Part.h -text src/include/H5_attribs.h -text -src/include/H5_inquiry.h -text src/include/H5hut.h -text src/include/grephdr -text src/include/h5core/h5_attribs.h -text @@ -530,6 +530,9 @@ test/H5Part/H5testF.f -text test/H5Part/H5testFpar.f90 -text test/H5Part/Makefile.am -text test/Makefile.am -text +test/h5b_read.c -text +test/h5b_test.c -text +test/h5b_write.c -text test/h5u_read.c -text test/h5u_test.c -text test/h5u_write.c -text diff --git a/Makefile.am b/Makefile.am index 2b4dc8a..49f9098 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,7 @@ SUBDIRS = \ - src + src \ + test \ + tools MAINTAINERCLEANFILES = \ config.h \ diff --git a/configure.ac b/configure.ac index 00f136e..c23465a 100644 --- a/configure.ac +++ b/configure.ac @@ -18,6 +18,12 @@ AC_CONFIG_HEADER(config.h) ################# --enable-xxx and --with-xxx Argument ######################## ############################################################################### +AC_ARG_ENABLE( + [debug], + [AC_HELP_STRING([--enable-debug], + [Compile with debug flags [default=no]])], + [USE_DEBUG=$enableval]) + AC_ARG_ENABLE( [64], [AC_HELP_STRING([--enable-64], @@ -54,6 +60,13 @@ AC_ARG_WITH( [path to HDF5 installation [default=""]])], [HDF5PATH=$withval], [HDF5PATH=""]) +AC_ARG_WITH( + [lustre], + [AC_HELP_STRING([--with-lustre], + [path to lustre user API [default=""]])], + [LUSTREPATH=$withval], [LUSTREPATH=""]) + + ############################################################################### ################# A SIMPLE WORK AROUND TO USE ENV. VARS ####################### ############################################################################### @@ -169,6 +182,15 @@ AC_DEFINE_UNQUOTED(MY_UNAME, "$uname", "") ######################## CONFIGURE LINE OPTIONS ############################### ############################################################################### +AC_MSG_CHECKING([if debug is enabled]) + +if test "X$USE_DEBUG" = "Xyes"; then + AC_MSG_RESULT([yes]) + CFLAGS="$CFLAGS -g" +else + AC_MSG_RESULT([no]) +fi + ############################ fortran enabled ################################## AC_MSG_CHECKING([if fortran interface enabled]) if test "X$USE_FORTRAN" = "Xyes"; then @@ -361,23 +383,7 @@ else P="$P /usr/local/hdf5" P="$P /usr/local/packages/hdf5" P="$P /apps/hdf5" - - if test "X$USE_PARALLEL" = "Xyes"; then - P="$P /usr/local/phdf5" - P="$P /usr/local/hdf5/hdf5_par" - if test "X$USE_64" = "Xyes"; then - P="$P /usr/common/usg/hdf5/64/default/parallel" - else - P="$P /usr/common/usg/hdf5/32/default/parallel" - fi - else - P="$P /usr/local/hdf5/hdf5_serial" - if test "X$USE_64" = "Xyes"; then - P="$P /usr/common/usg/hdf5/64/default/serial" - else - P="$P /usr/common/usg/hdf5/32/default/serial" - fi - fi + P="$P /opt/hdf5" fi PATH_Search HDF5ROOT "$P" include/hdf5.h if test -z "$HDF5ROOT"; then @@ -403,6 +409,27 @@ if test -n "$ZLIBROOT"; then LDFLAGS="$LDFLAGS -L$ZLIBROOT/lib" fi +AC_MSG_CHECKING([for lustre API ]) +AC_MSG_RESULT([]) +if test -n "${LUSTREPATH}"; then + P=${LUSTREPATH} +elif test -n "${LUSTRE_DIR}" ; then + P=${LUSTRE_DIR} +else + P='' + P="$P /usr" + P="$P /usr/local" + P="$P /usr/local/lustre" + P="$P /opt/lustre" +fi +PATH_Search LUSTREROOT "$P" usr/include/lustre/liblustreapi.h +if test -z "$LUSTREROOT"; then + AC_MSG_WARN([Couldn't locate the lustre API... building without support for lustre striping!]) +else + CFLAGS="$CFLAGS -DH5_USE_LUSTRE" +fi + + ############################################################################### ################# A SIMPLE WORK AROUND TO USE ENV. VARS ####################### ############################################################################### @@ -472,6 +499,7 @@ fi # If value is given, in addition assign it to variable. AC_SUBST(MPIROOT) AC_SUBST(HDF5ROOT) +AC_SUBST(LUSTREROOT) AC_SUBST(MPIINC) AC_SUBST(MPILIB) AC_SUBST(MPICC) @@ -531,5 +559,6 @@ AC_MSG_RESULT([MPILIB = $MPILIB]) AC_MSG_RESULT([MPIINC = $MPIINC]) AC_MSG_RESULT([MPIROOT = $MPIROOT]) AC_MSG_RESULT([HDF5ROOT = $HDF5ROOT]) +AC_MSG_RESULT([LUSTREROOT = $LUSTREROOT]) AC_MSG_RESULT([LDFLAGS = $LDFLAGS]) AC_MSG_RESULT([ ]) diff --git a/src/C/H5.c b/src/C/H5.c index 135d493..837e7a6 100644 --- a/src/C/H5.c +++ b/src/C/H5.c @@ -94,13 +94,13 @@ H5CheckFile ( Define format of the step names. - Example: ==H5FedDefineStepNameFormat( f, "Step", 6 )== defines step names + Example: ==H5SetStepNameFormat( f, "Step", 6 )== defines step names like ==Step#000042==. \return \c H5_SUCCESS or error code */ h5_err_t -H5DefineStepNameFormat ( +H5SetStepNameFormat ( h5_file_t* const f, /*!< Handle to file */ const char* name, /*!< Prefix */ const h5_int64_t width /*!< Width of the number */ @@ -140,7 +140,7 @@ H5GetStepNameFormat ( h5_err_t H5SetStep ( h5_file_t* const f, /*!< [in] Handle to open file */ - const h5_int64_t step /*!< [in] Step to set. */ + const h5_id_t step /*!< [in] Step to set. */ ) { SET_FNAME (f, __func__); @@ -163,6 +163,68 @@ H5GetStep ( return h5_get_step (f); } +/*! + \ingroup h5_inquiry + + Get the number of processors. + + \param[in] f File handle. + + \return Number of processors. + \return \c -1 on error. + */ +int +H5GetNumProcs ( + h5_file_t* const f + ) { + SET_FNAME (f, __func__); + return h5_get_num_procs(f); +} + +/*! + \ingroup h5part_c_api_read + + Get the number of time-steps that are currently stored in the file + \c f. + + It works for both reading and writing of files, but is probably + only typically used when you are reading. + + \param[in] f File handle. + + \return number of time-steps or error code +*/ +h5_size_t +H5GetNumSteps ( + h5_file_t* const f + ) { + + SET_FNAME (f, __func__); + + return h5_get_num_steps(f); +} + +/*! + \ingroup h5_inquiry + + Query whether a particular step already exists in the file. + + \param[in] f File handle. + \param[in] stepno Step number to query for existence + + \return true or false +*/ +h5_err_t +H5HasStep ( + h5_file_t* const f, + h5_id_t stepno + ) { + + SET_FNAME (f, __func__); + + return h5_has_step (f, stepno); +} + /*! \ingroup h5_c_api_general diff --git a/src/C/H5Block.c b/src/C/H5Block.c index d4ff936..d48b2b4 100644 --- a/src/C/H5Block.c +++ b/src/C/H5Block.c @@ -226,6 +226,9 @@ H5BlockGetNumFields ( Get the name, rank and dimensions of the field specified by the index \c idx. + \c elem_rank reports the rank of the elements in the field + (e.g. scalar or vector). + This function can be used to retrieve all fields bound to the current time-step by looping from \c 0 to the number of fields minus one. The number of fields bound to the current time-step @@ -239,22 +242,23 @@ H5BlockGetFieldInfo ( const h5_size_t idx, /*!< IN: index of field */ char *name, /*!< OUT: field name */ const h5_size_t len_name, /*!< IN: buffer size */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ) { SET_FNAME( f, __func__ ); - return h5b_get_field_info ( - f, idx, name, len_name, grid_rank, grid_dims, field_rank, type); + return h5b_get_field_info (f, + idx, name, len_name, field_rank, field_dims, elem_rank, type); } /*! \ingroup h5block_model Get the rank and dimensions of the field specified by its name. + See \ref H5BlockGetFieldInfo. \return \c H5_SUCCESS or error code */ @@ -262,16 +266,16 @@ h5_err_t H5BlockGetFieldInfoByName ( h5_file_t *const f, /*!< IN: file handle */ const char *name, /*!< IN: field name */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ - h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ) { SET_FNAME( f, __func__ ); - return h5b_get_field_info_by_name ( - f, name, grid_rank, grid_dims, field_rank, type ); + return h5b_get_field_info_by_name (f, + name, field_rank, field_dims, elem_rank, type ); } /********************** reading and writing attribute ************************/ diff --git a/src/C/H5Part.c b/src/C/H5Part.c index 3565646..30ad0cb 100644 --- a/src/C/H5Part.c +++ b/src/C/H5Part.c @@ -109,12 +109,12 @@ For further information contact: h5part h5_err_t H5PartSetNumParticles ( h5_file_t *f, /*!< [in] Handle to open file */ - h5_int64_t nparticles /*!< [in] Number of particles */ + h5_size_t nparticles /*!< [in] Number of particles */ ) { SET_FNAME( f, __func__ ); - h5_int64_t stride = 1; + h5_size_t stride = 1; return h5u_set_num_particles( f, nparticles, stride ); } @@ -148,8 +148,8 @@ H5PartSetNumParticles ( h5_err_t H5PartSetNumParticlesStrided ( h5_file_t *f, /*!< [in] Handle to open file */ - h5_int64_t nparticles, /*!< [in] Number of particles */ - h5_int64_t stride /*!< [in] Stride value (e.g. number of fields in the particle array) */ + h5_size_t nparticles, /*!< [in] Number of particles */ + h5_size_t stride /*!< [in] Stride value (e.g. number of fields in the particle array) */ ) { SET_FNAME( f, __func__ ); @@ -171,9 +171,9 @@ H5PartSetNumParticlesStrided ( \return \c H5_SUCCESS or error code */ h5_err_t -H5PartSetChunkSize ( +H5PartSetChunk ( h5_file_t *f, - h5_int64_t size + h5_size_t size ) { SET_FNAME( f, __func__ ); @@ -437,15 +437,13 @@ H5PartReadDataInt32 ( \return number of datasets in current timestep or error code */ -h5_int64_t +h5_ssize_t H5PartGetNumDatasets ( h5_file_t *f /*!< [in] Handle to open file */ ) { SET_FNAME( f, __func__ ); - CHECK_FILEHANDLE( f ); - return h5u_get_num_datasets(f); } @@ -459,19 +457,16 @@ H5PartGetNumDatasets ( \result \c H5_SUCCESS */ -h5_int64_t +h5_err_t H5PartGetDatasetName ( h5_file_t *f, /*!< [in] Handle to open file */ - const h5_int64_t idx, /*!< [in] Index of the dataset */ + const h5_id_t idx, /*!< [in] Index of the dataset */ char *name, /*!< [out] Name of dataset */ - const h5_int64_t len /*!< [in] Size of buffer \c name */ + const h5_size_t len /*!< [in] Size of buffer \c name */ ) { SET_FNAME( f, __func__ ); - CHECK_FILEHANDLE ( f ); - //CHECK_TIMEGROUP ( f ); - return h5u_get_dataset_info(f, idx, name, len, NULL, NULL); } @@ -481,54 +476,53 @@ H5PartGetDatasetName ( Gets the name, type and number of elements of a dataset based on its index in the current timestep. - Type is one of the following macros: + Type is one of the following values: - - \c h5_float64 (for \c h5part_float64_t) - - \c h5_float32 (for \c h5part_float32_t) - - \c h5_int64 (for \c h5part_int64_t) - - \c h5_int32 (for \c h5part_int32_t) - - \c H5PART_CHAR (for \c char) - - \c H5PART_STRING (for \c char*) + - \c H5_FLOAT64_T (for \c h5_float64_t) + - \c H5_FLOAT32_T (for \c h5_float32_t) + - \c H5_INT64_T (for \c h5_int64_t) + - \c H5_INT32_T (for \c h5_int32_t) \return \c H5_SUCCESS */ -h5_int64_t +h5_err_t H5PartGetDatasetInfo ( h5_file_t *f, /*!< [in] Handle to open file */ - const h5_int64_t idx,/*!< [in] Index of the dataset */ + const h5_id_t idx, /*!< [in] Index of the dataset */ char *dataset_name, /*!< [out] Name of dataset */ - const h5_int64_t len_dataset_name, + const h5_size_t len_dataset_name, /*!< [in] Size of buffer \c dataset_name */ h5_int64_t *type, /*!< [out] Type of data in dataset */ - h5_int64_t *nelem /*!< [out] Number of elements. */ + h5_size_t *nelem /*!< [out] Number of elements. */ ) { SET_FNAME( f, __func__ ); - CHECK_FILEHANDLE ( f ); - //CHECK_TIMEGROUP ( f ); - return h5u_get_dataset_info(f, idx, dataset_name, len_dataset_name, type, nelem); } /*! \ingroup h5part_model - This function returns the number of particles in the first dataset of - the current timestep (or in the first timestep if none has been set). + This function returns the number of particles in this processor's view, + if a view has been set. - If you have neither set the number of particles (read or write) - nor set a view (read-only), then this returns the total number of - elements on disk of the first dataset if is exists. Otherwise, - it returns 0. + If not, it returns the total number of particles across all processors + from the last \ref H5PartSetNumParticles call. - If you have set a view, this return the number of particles - in the view. + If you have neither set the number of particles + nor set a view, then this returns the total number of + particles in the first data set of the current time step. + Note that H5Part assumes that all data sets within a given time step + have the same number of particles (although the number particles can + vary across time steps). + + If none of these conditions are met, an error is thrown. \return number of particles in current timestep or an error code. */ -h5_int64_t +h5_ssize_t H5PartGetNumParticles ( h5_file_t *f /*!< [in] Handle to open file */ ) { @@ -556,7 +550,7 @@ H5PartResetView ( /*! \ingroup h5part_model */ -h5_int64_t +h5_err_t H5PartHasView ( h5_file_t *f /*!< [in] Handle to open file */ ) { @@ -569,15 +563,15 @@ H5PartHasView ( /*! \ingroup h5part_model - For parallel I/O or for subsetting operations on the datafile, the - \c H5PartSetView() function allows you to define a subset of the total + For parallel I/O or for subsetting operations on the datafile, + this function allows you to define a subset of the total particle dataset to operate on. The concept of "view" works for both serial and for parallel I/O. The "view" will remain in effect until a new view is set, or the number of particles in a dataset changes, or the view is "unset" by calling \c H5PartSetView(file,-1,-1); - Before you set a view, the \c H5PartGetNumParticles() will return the + Before you set a view, \ref H5PartGetNumParticles will return the total number of particles in the current time-step (even for the parallel reads). However, after you set a view, it will return the number of particles contained in the view. @@ -620,8 +614,8 @@ H5PartSetView ( h5_err_t H5PartSetViewIndices ( h5_file_t *f, /*!< [in] Handle to open file */ - const h5_int64_t *indices, /*!< [in] List of indices */ - h5_int64_t nelems /*!< [in] Size of list */ + const h5_id_t *indices, /*!< [in] List of indices */ + h5_size_t nelems /*!< [in] Size of list */ ) { SET_FNAME( f, __func__ ); diff --git a/src/C/H5_attribs.c b/src/C/H5_attribs.c index 835b1bf..5e7621d 100644 --- a/src/C/H5_attribs.c +++ b/src/C/H5_attribs.c @@ -161,7 +161,7 @@ H5WriteStepAttribFloat64 ( f, H5_ATTRIB_STEP, name, - H5T_NATIVE_FLOAT, + H5T_NATIVE_DOUBLE, values, nelems ); } diff --git a/src/C/H5_inquiry.c b/src/C/H5_inquiry.c deleted file mode 100644 index b4fe679..0000000 --- a/src/C/H5_inquiry.c +++ /dev/null @@ -1,89 +0,0 @@ -/* - Copyright 2007-2008 - Paul Scherrer Institut, Villigen, Switzerland; - Benedikt Oswald; - Achim Gsell - All rights reserved. - - Authors - Achim Gsell - - Warning - This code is under development. - - */ -/*! - \ingroup h5f_c_api - \defgroup h5_inquiry -*/ - -#include "h5core/h5_core.h" -#include "H5.h" - -/*! - \ingroup h5_inquiry - - Get the number of compute nodes. - - \param[in] f File handle. - - \return Number of compute notes. - \return \c -1 on error. - */ -int -H5GetNumNodes ( - h5_file_t* const f - ) { - SET_FNAME (f, __func__); - CHECK_FILEHANDLE (f); - return h5_get_num_procs(f); -} - -/*! - \ingroup h5part_c_api_read - - Get the number of time-steps that are currently stored in the file - \c f. - - It works for both reading and writing of files, but is probably - only typically used when you are reading. - - \param[in] f File handle. - - \return number of time-steps or error code -*/ -h5_size_t -H5GetNumSteps ( - h5_file_t* const f - ) { - - SET_FNAME (f, __func__); - CHECK_FILEHANDLE (f); - - return h5_get_num_steps(f); -} - -/*! - \ingroup h5_inquiry - - Query whether a particular step already exists in the file. - - \param[in] f File handle. - \param[in] stepno Step number to query for existence - - \return true or false -*/ -h5_err_t -H5HasStep ( - h5_file_t* const f, - h5_id_t stepno - ) { - - SET_FNAME (f, __func__); - CHECK_FILEHANDLE (f); - - return h5_has_step (f, stepno); -} - - - diff --git a/src/C/Makefile.am b/src/C/Makefile.am index 50103a2..a6c2e0c 100644 --- a/src/C/Makefile.am +++ b/src/C/Makefile.am @@ -17,7 +17,6 @@ EXTRA_LIBRARIES = libH5hutC.a include_HEADERS = \ ../include/H5hut.h \ ../include/H5.h \ - ../include/H5_inquiry.h \ ../include/H5_attribs.h \ ../include/H5Part.h \ ../include/H5Block.h \ @@ -35,7 +34,6 @@ EXTRA_HEADERS = libH5hutC_a_SOURCES = \ H5.c \ H5_attribs.c \ - H5_inquiry.c \ H5Part.c \ H5Block.c \ H5Block_readwrite.c \ diff --git a/src/C/generate-h5b-readwrite.py b/src/C/generate-h5b-readwrite.py index f9f59d8..812f27a 100755 --- a/src/C/generate-h5b-readwrite.py +++ b/src/C/generate-h5b-readwrite.py @@ -14,6 +14,8 @@ h_tail = """ """ fc_head = """ +#include + #include "H5hut.h" #include "Underscore.h" diff --git a/src/Fortran/H5Block_F.c b/src/Fortran/H5Block_F.c index d93b3d5..dde8256 100755 --- a/src/Fortran/H5Block_F.c +++ b/src/Fortran/H5Block_F.c @@ -277,7 +277,7 @@ h5bl_readfieldattrib_string ( char *field_name2 = h5_strdupfor2c ( field_name, l_field_name ); char *attrib_name2 = h5_strdupfor2c ( attrib_name, l_attrib_name ); - h5_err_t herr = H5BlockReadFieldAttrib ( + h5_err_t herr = H5BlockReadFieldAttribString ( filehandle, field_name2, attrib_name2, attrib_value ); h5_strc2for ( attrib_value, l_attrib_value ); diff --git a/src/Fortran/H5Block_readwrite_F.c b/src/Fortran/H5Block_readwrite_F.c index 4fa1765..fcbccd3 100644 --- a/src/Fortran/H5Block_readwrite_F.c +++ b/src/Fortran/H5Block_readwrite_F.c @@ -1,4 +1,6 @@ +#include + #include "H5hut.h" #include "Underscore.h" diff --git a/src/Fortran/H5Part_F.c b/src/Fortran/H5Part_F.c index 2243a25..b463f15 100755 --- a/src/Fortran/H5Part_F.c +++ b/src/Fortran/H5Part_F.c @@ -1,3 +1,5 @@ +#include + #include "H5hut.h" #include "Underscore.h" @@ -97,16 +99,6 @@ h5pt_setnpoints_strided ( /*==============Reading Data Characteristics============*/ -h5_err_t -h5pt_getnsteps ( - const h5_int64_t *f - ) { - - h5_file_t *filehandle = (h5_file_t*)(size_t)*f; - - return H5PartGetNumSteps ( filehandle ); -} - h5_err_t h5pt_getndatasets ( const h5_int64_t *f @@ -140,7 +132,7 @@ h5pt_getdatasetname ( h5_int64_t herr = H5PartGetDatasetName ( filehandle, *index, name, l_name ); - _h5_strc2for ( name, l_name ); + h5_strc2for ( name, l_name ); return herr; } diff --git a/src/h5core/Makefile.am b/src/h5core/Makefile.am index 2468144..8152d6f 100644 --- a/src/h5core/Makefile.am +++ b/src/h5core/Makefile.am @@ -38,6 +38,7 @@ EXTRA_HEADERS = \ h5_qsort_private.h \ h5_readwrite_private.h \ h5_syscall_private.h \ + h5_lustre_private.h \ h5u_errorhandling_private.h \ h5b_errorhandling_private.h \ h5b_model_private.h \ @@ -64,16 +65,16 @@ nodist_include_HEADERS = #OBJEXT = o # What to build... Will be determined by configure script. -lib_LIBRARIES = libH5Core.a +lib_LIBRARIES = libH5hut.a # Listing of all possible targets that I may build. -EXTRA_LIBRARIES = libH5Core.a +EXTRA_LIBRARIES = libH5hut.a # Header files that I wish to install in $(prefix)/include include_HEADERS = # Listing of sources -libH5Core_a_SOURCES = \ +libH5hut_a_SOURCES = \ h5_attribs.c \ h5_errorhandling.c \ h5_fcmp.c \ @@ -86,6 +87,7 @@ libH5Core_a_SOURCES = \ h5_qsort_r.c \ h5_readwrite.c \ h5_syscall.c \ + h5_lustre.c \ h5u_readwrite.c \ h5b_readwrite.c \ h5u_model.c \ @@ -110,12 +112,12 @@ libH5Core_a_SOURCES = \ h5t_store_trim.c \ h5t_tags.c -libH5Core_a_DEPENDENCIES = $(EXTRA_HEADERS) +libH5hut_a_DEPENDENCIES = $(EXTRA_HEADERS) -all: ../lib/libH5Core.a +all: ../lib/libH5hut.a -../lib/libH5Core.a: libH5Core.a +../lib/libH5hut.a: libH5hut.a -cp $^ $@ -$(libH5Core_a_OBJECTS): $(libH5Core_a_DEPENDENCIES) +$(libH5hut_a_OBJECTS): $(libH5hut_a_DEPENDENCIES) diff --git a/src/h5core/h5_attribs.c b/src/h5core/h5_attribs.c index 63c66f4..7150ffb 100644 --- a/src/h5core/h5_attribs.c +++ b/src/h5core/h5_attribs.c @@ -31,10 +31,12 @@ h5_read_attrib ( const hid_t attrib_type, /*!< HDF5 type of attribute */ void* const attrib_value /*!< OUT: attribute value */ ) { + + if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f ); + hid_t attrib_id; hid_t space_id; hid_t type_id; - hid_t mytype; hid_t id; TRY( _get_hdf5_obj_id(f, mode, &id) ); @@ -56,7 +58,7 @@ h5_read_attrib ( TRY( space_id = h5priv_get_hdf5_attribute_dataspace (f, attrib_id) ); TRY( h5priv_read_hdf5_attribute (f, attrib_id, type_id, attrib_value) ); TRY( h5priv_close_hdf5_dataspace(f, space_id) ); - TRY( h5priv_close_hdf5_type(f, mytype) ); + TRY( h5priv_close_hdf5_type(f, type_id) ); TRY( h5priv_close_hdf5_attribute (f, attrib_id) ); return H5_SUCCESS; @@ -78,6 +80,10 @@ h5_write_attrib ( const void* attrib_value, /*!< value of attribute */ const hsize_t attrib_nelem /*!< number of elements (dimension) */ ) { + + if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f ); + CHECK_WRITABLE_MODE( f ); + hid_t space_id; hid_t attrib_id; hid_t type_id; @@ -131,6 +137,9 @@ h5_get_attrib_info ( h5_int64_t* attrib_type, /*!< OUT: H5 type of attribute */ h5_size_t* attrib_nelem /*!< OUT: number of elements */ ) { + + if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f ); + hid_t attrib_id; hid_t mytype; hid_t space_id; @@ -178,7 +187,7 @@ h5_get_num_attribs ( h5_file_t *const f, /*!< handle to open file */ const char mode /*!< FILE or STEP flag */ ) { - CHECK_FILEHANDLE (f); + if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f ); hid_t id; TRY( _get_hdf5_obj_id(f, mode, &id) ); return h5priv_get_num_hdf5_attribute (f, id); diff --git a/src/h5core/h5_core_private.h b/src/h5core/h5_core_private.h index 83600fe..92a4ada 100644 --- a/src/h5core/h5_core_private.h +++ b/src/h5core/h5_core_private.h @@ -1,14 +1,16 @@ #ifndef __H5_CORE_PRIVATE_H #define __H5_CORE_PRIVATE_H -#define H5_DATANAME_LEN 128 -#define H5_STEPNAME_LEN 128 +/* WARNING! Changing these values will alter the data model and introduce + * file incompatibilities with previous versions. */ +#define H5_DATANAME_LEN 64 +#define H5_STEPNAME_LEN 64 #define H5_STEPNAME "Step" #define H5_STEPWIDTH 1 #define H5_BLOCKNAME "Block" #define H5_BLOCKNAME_X "0" -#define H5_BLOCKNAME_Y "0" -#define H5_BLOCKNAME_Z "0" +#define H5_BLOCKNAME_Y "1" +#define H5_BLOCKNAME_Z "2" #include "h5_types_private.h" @@ -20,6 +22,9 @@ #include "h5_qsort_private.h" #include "h5_readwrite_private.h" #include "h5_syscall_private.h" +#ifdef H5_USE_LUSTRE +#include "h5_lustre_private.h" +#endif #include "h5b_types_private.h" #include "h5u_types_private.h" diff --git a/src/h5core/h5_errorhandling.c b/src/h5core/h5_errorhandling.c index 890ef8d..844dda0 100644 --- a/src/h5core/h5_errorhandling.c +++ b/src/h5core/h5_errorhandling.c @@ -6,7 +6,7 @@ #include "h5_core_private.h" static h5_errorhandler_t h5priv_errhandler = h5_report_errorhandler; -static h5_int32_t h5priv_debug_level = 0; +static h5_int32_t h5priv_debug_level = 1; /*! \ingroup h5_core diff --git a/src/h5core/h5_hdf5.c b/src/h5core/h5_hdf5.c index 834e170..0c53e1d 100644 --- a/src/h5core/h5_hdf5.c +++ b/src/h5core/h5_hdf5.c @@ -1,3 +1,4 @@ +#include #include #include @@ -55,7 +56,6 @@ h5priv_open_group ( const char* const group_name ) { hid_t group_id; - herr_t herr = H5Gget_objinfo (loc_id, group_name, 1, NULL); /* check access modes: @@ -66,7 +66,9 @@ h5priv_open_group ( H5_O_APPEND x x (append datasets to an existing group) */ - if (herr >= 0) { + h5_err_t exists; + TRY( exists = h5priv_hdf5_link_exists(f, loc_id, group_name) ); + if (exists > 0) { h5_info ( f, "Opening group %s/%s.", @@ -87,7 +89,7 @@ h5priv_open_group ( return h5_error ( f, H5_ERR_HDF5, - "Cannot open group \"%s/%s\".", + "Cannot open or create group %s/%s.", h5_get_objname (loc_id), group_name); @@ -574,7 +576,7 @@ h5priv_close_hdf5_dataspace ( h5_file_t* const f, const hid_t dataspace_id ) { - if (dataspace_id == 0 || dataspace_id == -1 || dataspace_id == H5S_ALL) + if (dataspace_id <= 0 || dataspace_id == H5S_ALL) return H5_SUCCESS; herr_t herr = H5Sclose (dataspace_id); @@ -582,8 +584,7 @@ h5priv_close_hdf5_dataspace ( return h5_error( f, H5_ERR_HDF5, - "Cannot terminate access to dataspace \"%s\".", - h5_get_objname (dataspace_id)); + "Cannot terminate access to dataspace!"); return H5_SUCCESS; } @@ -1202,12 +1203,71 @@ h5priv_delete_hdf5_link ( typedef struct op_data { int queried_idx; int cnt; - H5L_type_t type; + H5O_type_t type; char *name; size_t len; char *prefix; + h5_file_t *f; } op_data_t; +static H5O_type_t +_iter_op_get_obj_type ( + h5_file_t *const f, + const hid_t g_id, + const char* name, + const H5L_info_t* info + ) { + herr_t herr; + H5O_info_t objinfo; + + if ( info->type == H5L_TYPE_EXTERNAL ) { + char *buf; + TRY( buf = h5priv_alloc(f, NULL, info->u.val_size) ); + + herr = H5Lget_val(g_id, name, buf, + info->u.val_size, H5P_DEFAULT); + if ( herr < 0 ) + return h5_error(f, + H5_ERR_HDF5, + "Can't get external link for object '%s'!", + name); + + const char *filename; + const char *objname; + herr = H5Lunpack_elink_val(buf, info->u.val_size, 0, + &filename, &objname); + if ( herr < 0 ) + return h5_error(f, + H5_ERR_HDF5, + "Can't unpack external link for object '%s'!", + name); + + h5_debug(f, + "Followed external link to file '%s' / object '%s'.", + filename, objname); + + free(buf); + + hid_t obj_id = H5Oopen(g_id, name, H5P_DEFAULT); + if ( obj_id < 0 ) + return h5_error(f, + H5_ERR_HDF5, + "Can't open external link for object '%s'!", + name); + herr = H5Oget_info(obj_id, &objinfo); + } + else { // H5L_TYPE_HARD + herr = H5Oget_info_by_name(g_id, name, &objinfo, H5P_DEFAULT); + } + + if ( herr < 0 ) + return h5_error(f, + H5_ERR_HDF5, + "Can't query object with name '%s'!", name); + + return objinfo.type; +} + static herr_t iter_op_count ( hid_t g_id, @@ -1216,10 +1276,9 @@ iter_op_count ( void* _op_data ) { op_data_t* op_data = (op_data_t*)_op_data; - - if (info->type != op_data->type) { - return 0; - } + H5O_type_t type; + TRY( type = _iter_op_get_obj_type(op_data->f, g_id, name, info) ); + if ( type != op_data->type ) return 0; op_data->cnt++; return 0; } @@ -1232,9 +1291,9 @@ iter_op_idx ( void* _op_data ) { op_data_t* op_data = (op_data_t*)_op_data; - if (info->type != op_data->type) { - return 0; - } + H5O_type_t type; + TRY( type = _iter_op_get_obj_type(op_data->f, g_id, name, info) ); + if ( type != op_data->type ) return 0; op_data->cnt++; /* stop iterating if index is equal cnt */ if (op_data->queried_idx == op_data->cnt) { @@ -1253,14 +1312,13 @@ iter_op_count_match ( void* _op_data ) { op_data_t* op_data = (op_data_t*)_op_data; - if (info->type != op_data->type) { - return 0; - } + H5O_type_t type; + TRY( type = _iter_op_get_obj_type(op_data->f, g_id, name, info) ); + if ( type != op_data->type ) return 0; /* count if prefix matches */ if (strncmp (name, op_data->prefix, strlen(op_data->prefix)) == 0) { op_data->cnt++; } - return 0; } @@ -1271,7 +1329,7 @@ h5_get_num_hdf5_groups ( ) { op_data_t op_data; memset (&op_data, 0, sizeof (op_data)); - op_data.type = (H5L_type_t)H5G_GROUP; + op_data.type = H5O_TYPE_GROUP; hsize_t start_idx = 0; herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC, &start_idx, @@ -1294,7 +1352,8 @@ h5_get_num_hdf5_groups_matching_prefix ( ) { op_data_t op_data; memset (&op_data, 0, sizeof (op_data)); - op_data.type = (H5L_type_t)H5G_GROUP; + op_data.f = f; + op_data.type = H5O_TYPE_GROUP; op_data.prefix = prefix; hsize_t start_idx = 0; herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC, @@ -1316,13 +1375,16 @@ h5_get_hdf5_groupname_by_idx ( hid_t loc_id, hsize_t idx, char *name, - size_t size + size_t len ) { op_data_t op_data; memset (&op_data, 0, sizeof (op_data)); - op_data.type = (H5L_type_t)H5G_GROUP; + op_data.f = f; + op_data.type = H5O_TYPE_GROUP; op_data.cnt = -1; op_data.queried_idx = idx; + op_data.name = name; + op_data.len = len; hsize_t start_idx = 0; herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC, &start_idx, @@ -1334,7 +1396,6 @@ h5_get_hdf5_groupname_by_idx ( "Cannot get name of group with index \"%lu\" in \"%s\".", (long unsigned int)idx, h5_get_objname (loc_id)); } - strncpy (name, op_data.name, size); return H5_SUCCESS; } @@ -1345,7 +1406,8 @@ h5_get_num_hdf5_datasets ( ) { op_data_t op_data; memset (&op_data, 0, sizeof (op_data)); - op_data.type = (H5L_type_t)H5G_DATASET; + op_data.f = f; + op_data.type = H5O_TYPE_DATASET; hsize_t start_idx = 0; herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC, &start_idx, @@ -1369,14 +1431,16 @@ h5_get_hdf5_datasetname_by_idx ( hid_t loc_id, hsize_t idx, char *name, - size_t size + size_t len ) { - return H5_SUCCESS; op_data_t op_data; memset (&op_data, 0, sizeof (op_data)); - op_data.type = (H5L_type_t)H5G_DATASET; + op_data.f = f; + op_data.type = H5O_TYPE_DATASET; op_data.cnt = -1; op_data.queried_idx = idx; + op_data.name = name; + op_data.len = len; hsize_t start_idx = 0; herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC, &start_idx, @@ -1388,7 +1452,6 @@ h5_get_hdf5_datasetname_by_idx ( "Cannot get name of dataset with index \"%lu\" in \"%s\".", (long unsigned int)idx, h5_get_objname (loc_id)); } - strncpy (name, op_data.name, size); return H5_SUCCESS; } diff --git a/src/h5core/h5_lustre.c b/src/h5core/h5_lustre.c new file mode 100644 index 0000000..ff86e78 --- /dev/null +++ b/src/h5core/h5_lustre.c @@ -0,0 +1,155 @@ +#ifdef H5_USE_LUSTRE + +#include +#include +#include +#include +#include +#define __USE_GNU +#include +#undef __USE_GNU +#include + +#include "h5core/h5_core.h" +#include "h5_core_private.h" + +#define MSG_HEADER "optimize for lustre: " + +static void +_print_stripe_info(struct lov_user_md *lum) +{ + fprintf (stderr, "lmm_magic: %u\n", (unsigned)lum->lmm_magic); + fprintf (stderr, "lmm_pattern: %u\n", (unsigned)lum->lmm_pattern); + fprintf (stderr, "lmm_object_id: %lu\n", (unsigned long)lum->lmm_object_id); + fprintf (stderr, "lmm_object_gr: %lu\n", (unsigned long)lum->lmm_object_gr); + fprintf (stderr, "lmm_stripe_size: %u\n", (unsigned)lum->lmm_stripe_size); + fprintf (stderr, "lmm_stripe_count: %u\n", (unsigned)lum->lmm_stripe_count); + fprintf (stderr, "lmm_stripe_offset: %u\n", (unsigned)lum->lmm_stripe_offset); +} + +static ssize_t +_get_lustre_stripe_size ( const char *path ) +{ + size_t nbytes = sizeof(struct lov_user_md) + + INIT_ALLOC_NUM_OSTS * sizeof(struct lov_user_ost_data); + struct lov_user_md *lum; + TRY( lum = h5priv_alloc(f, NULL, nbytes) ); + lum->lmm_magic = LOV_USER_MAGIC; + + int fd = open64(path, O_RDONLY); + if (fd < 0) { + extern int errno; + if (errno == EINVAL) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "open64: a flag is invalid!"); + else if (errno == EACCES) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "open64: access denied or file does not exist!"); + else if (errno == ENAMETOOLONG) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "open64: path is too long!"); + else + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "open64: unspecific error!"); + return -1; + } + + int ret = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum); + if (ret == -1) { + extern int errno; + if (errno == EBADF) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "ioctl: bad file handle!"); + else if (errno == EINVAL) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "ioctl: invalid argument!"); + else if (errno == EIO) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "ioctl: physical I/O problem!"); + else if (errno == ENOTTY) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "ioctl: file handle does not accept control functions!"); + else if (errno == ENODEV) + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "ioctl: driver doesn't support control functions!"); + else + h5_error(f, H5_ERR_INTERNAL, MSG_HEADER + "ioctl: unspecific error!"); + return -1; + } + + close(fd); + + if ( f->myproc == 0 && h5_get_debuglevel() >= 5 ) + _print_stripe_info(lum); + + ssize_t stripe_size = (ssize_t)lum->lmm_stripe_size; + free(lum); + + return stripe_size; +} + +herr_t +h5_optimize_for_lustre ( + h5_file_t *const f, + const char *filename + ) { + + ssize_t stripe_size; + if ( f->myproc == 0 ) + { + char *path = malloc(strlen(filename)+4); + strcpy(path, filename); + /* check for existing file */ + FILE *test = fopen(path, "r"); + if (!test) { + /* use directory as path */ + int i = strlen(path)-1; + while (i >= 0) { + if (path[i] != '/') path[i] = '\0'; + else break; + i--; + } + if (strlen(path) == 0) sprintf(path, "."); + } + else fclose(test); + + stripe_size = _get_lustre_stripe_size(path); + + free(path); + } + + TRY( h5priv_mpi_bcast(f, &stripe_size, 1, MPI_LONG_LONG, 0, f->comm) ); + h5_info(f, "Found lustre stripe size of %lld bytes", (long long)stripe_size); + + hsize_t btree_ik = (stripe_size - 4096) / 96; + hsize_t btree_bytes = 64 + 96*btree_ik; + h5_info(f, + "Setting HDF5 btree ik to %lld (= %lld bytes at rank 3)", + (long long)btree_ik, (long long)btree_bytes); + TRY( h5priv_set_hdf5_btree_ik_property(f, f->create_prop, btree_ik) ); + + /* set alignment to lustre stripe size */ + TRY( h5priv_set_hdf5_alignment_property(f, + f->access_prop, 0, stripe_size) ); + + if (_verbose) { + fprintf(MSG_STREAM, MSG_HEADER "disabling metadata cache flushes.\n"); + } + /* disable metadata cache flushes */ + /* defer metadata writes */ + H5AC_cache_config_t config; + config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + TRY( H5Pget_mdc_config( fapl_id, &config ) ); + config.set_initial_size = 1; + config.initial_size = 16 * 1024 * 1024; + config.evictions_enabled = 0; + config.incr_mode = H5C_incr__off; + config.decr_mode = H5C_decr__off; + config.flash_incr_mode = H5C_flash_incr__off; + TRY( H5Pset_mdc_config( fapl_id, &config ) ); + + return H5_SUCCESS; +} + +#endif // H5_USE_LUSTRE diff --git a/src/h5core/h5_lustre_private.h b/src/h5core/h5_lustre_private.h new file mode 100644 index 0000000..921a6a2 --- /dev/null +++ b/src/h5core/h5_lustre_private.h @@ -0,0 +1,10 @@ +#ifndef __H5_LUSTRE_PRIVATE_H +#define __H5_LUSTRE_PRIVATE_H + +h5_err_t +h5_optimize_for_lustre ( + h5_file_t *const f, + const char *filename + ); + +#endif diff --git a/src/h5core/h5_mpi.c b/src/h5core/h5_mpi.c index 2159046..ea22865 100644 --- a/src/h5core/h5_mpi.c +++ b/src/h5core/h5_mpi.c @@ -50,6 +50,29 @@ h5priv_mpi_send( return H5_SUCCESS; } +h5_err_t +h5priv_mpi_bcast ( + h5_file_t* const f, + void* buf, + const int count, + const MPI_Datatype type, + const int root, + const MPI_Comm comm + ) { + int err = MPI_Bcast( + buf, + count, + type, + root, + comm + ); + if (err != MPI_SUCCESS) + return h5_error (f, H5_ERR_MPI, "Cannot perform broadcast"); + return H5_SUCCESS; +} + + + h5_err_t h5priv_mpi_sum ( h5_file_t* const f, diff --git a/src/h5core/h5_mpi_private.h b/src/h5core/h5_mpi_private.h index f2ebd08..bbc22c8 100644 --- a/src/h5core/h5_mpi_private.h +++ b/src/h5core/h5_mpi_private.h @@ -24,6 +24,16 @@ h5priv_mpi_send( const MPI_Comm comm ); +h5_err_t +h5priv_mpi_bcast ( + h5_file_t* const f, + void* buf, + const int count, + const MPI_Datatype type, + const int root, + const MPI_Comm comm + ); + h5_err_t h5priv_mpi_sum ( h5_file_t* const f, diff --git a/src/h5core/h5_openclose.c b/src/h5core/h5_openclose.c index 30d9142..6a0c68e 100644 --- a/src/h5core/h5_openclose.c +++ b/src/h5core/h5_openclose.c @@ -3,9 +3,6 @@ #include #include #include -#if H5_LUSTRE -#include -#endif #include "h5core/h5_core.h" #include "h5_core_private.h" @@ -27,7 +24,7 @@ h5_check_filehandle ( h5_file_t* const f /*!< filehandle to check validity of */ ) { - if (f == NULL || f->file == 0 || f->u == NULL || f->b == NULL || f->t == NULL) { + if (f == NULL || f->file < 0 || f->u == NULL || f->b == NULL || f->t == NULL) { return h5_error ( f, H5_ERR_BADFD, @@ -59,13 +56,14 @@ h5priv_error_handler ( \return H5_SUCCESS or error code */ -static h5_int64_t +static h5_err_t h5upriv_open_file ( h5_file_t* const f /*!< IN: file handle */ ) { TRY( f->u = (h5u_fdata_t*)h5priv_alloc (f, NULL, sizeof (*f->u)) ); h5u_fdata_t *u = f->u; + u->shape = -1; u->diskshape = H5S_ALL; u->memshape = H5S_ALL; u->viewstart = -1; @@ -86,7 +84,7 @@ h5upriv_open_file ( \return H5_SUCCESS or error code */ -static h5_int64_t +static h5_err_t h5bpriv_open_file ( h5_file_t * const f /*!< IN: file handle */ ) { @@ -99,15 +97,13 @@ h5bpriv_open_file ( b = f->b; memset (b, 0, sizeof (*b)); - size_t size = f->nprocs * sizeof (b->user_layout[0]); - TRY( b->user_layout = h5priv_alloc (f, NULL, size) ); - size = f->nprocs * sizeof (b->write_layout[0]); - TRY( b->write_layout = h5priv_alloc (f, NULL, size) ); - size_t n = sizeof (struct h5b_partition) / sizeof (h5_int64_t); TRY( h5priv_mpi_type_contiguous(f, n, MPI_LONG_LONG, &b->partition_mpi_t) ); + memset (b->user_layout, 0, sizeof(*b->user_layout)); + memset (b->write_layout, 0, sizeof(*b->write_layout)); + b->shape = -1; b->diskshape = -1; b->memshape = -1; @@ -177,33 +173,10 @@ h5priv_open_file ( f->xfer_prop, H5FD_MPIO_COLLECTIVE) ); } } - - /* defer metadata writes */ - H5AC_cache_config_t config; - config.version = H5AC__CURR_CACHE_CONFIG_VERSION; - TRY( h5priv_get_hdf5_mdc_property(f, f->access_prop, &config) ); - config.set_initial_size = 1; - config.initial_size = 16 * 1024 * 1024; - config.evictions_enabled = 0; - config.incr_mode = H5C_incr__off; - config.decr_mode = H5C_decr__off; - config.flash_incr_mode = H5C_flash_incr__off; - TRY( h5priv_set_hdf5_mdc_property(f, f->access_prop, &config) ); - #endif /* PARALLEL_IO */ -#if H5_LUSTRE -// set alignment - lov_user_md lum; - llapi_file_get_stripe(filename, &lum); - hsize_t stripe_size = (hsize_t)lum.lmm_stripe_size; - h5info(f, "Found lustre stripe size of %lld bytes", (long long)stripe_size); - TRY( h5priv_set_hdf5_alignment_property(f, - f->access_prop, 0, stripe_size) ); - hsize_t btree_ik = (stripe_size - 4096) / 96; - hsize_t btree_bytes = 64 + 96*btree_ik; - h5info(f, "Using %lld bytes for HDF5 btree", (long long)btree_bytes); - TRY( h5priv_set_hdf5_btree_ik_property(f, f->create_prop, btree_ik) ); +#ifdef H5_USE_LUSTRE + TRY( h5_optimize_for_lustre(f, filename) ); #endif if (flags & H5_O_RDONLY) { @@ -240,7 +213,7 @@ h5priv_open_file ( H5_ERR_HDF5, "Cannot open file \"%s\" with mode \"%d\"", filename, flags); - TRY( f->root_gid = h5priv_open_group (f, f->file, "/" )); + TRY( f->root_gid = h5priv_open_hdf5_group (f, f->file, "/" ) ); f->mode = flags; f->step_gid = -1; f->throttle = 0; @@ -300,25 +273,16 @@ h5_open_file ( \return H5_SUCCESS or error code */ -static h5_int64_t +static h5_err_t h5upriv_close_file ( h5_file_t* const f /*!< file handle */ ) { struct h5u_fdata* u = f->u; f->__errno = H5_SUCCESS; - if(u->shape != H5S_ALL) { - TRY( h5priv_close_hdf5_dataspace (f, u->shape) ); - u->shape = 0; - } - if(u->diskshape != H5S_ALL) { - TRY( h5priv_close_hdf5_dataspace (f, u->diskshape) ); - u->diskshape = 0; - } - if(u->memshape != H5S_ALL) { - TRY( h5priv_close_hdf5_dataspace (f, u->memshape) ); - u->memshape = 0; - } + TRY( h5priv_close_hdf5_dataspace (f, u->shape) ); + TRY( h5priv_close_hdf5_dataspace (f, u->diskshape) ); + TRY( h5priv_close_hdf5_dataspace (f, u->memshape) ); TRY( h5priv_close_hdf5_property (f, u->dcreate_prop) ); free (f->u); f->u = NULL; @@ -336,13 +300,14 @@ h5upriv_close_file ( \return H5_SUCCESS or error code */ -static h5_int64_t +static h5_err_t h5bpriv_close_file ( h5_file_t* const f /*!< IN: file handle */ ) { struct h5b_fdata* b = f->b; TRY( h5priv_close_hdf5_group (f, b->block_gid) ); + TRY( h5priv_close_hdf5_group (f, b->field_gid) ); TRY( h5priv_close_hdf5_dataspace (f, b->shape) ); TRY( h5priv_close_hdf5_dataspace (f, b->diskshape) ); TRY( h5priv_close_hdf5_dataspace (f, b->memshape) ); @@ -490,25 +455,6 @@ h5_get_num_steps( f->prefix_step_name); } -/*! - \ingroup h5_core_filehandling - - Check whether step with number \c stepno exists. - - \return True (value != 0) if step with \c stepno exists. - \return False (0) otherwise -*/ -h5_err_t -h5_has_step ( - h5_file_t* const f, /*!< file handle */ - h5_id_t stepno /*!< step number to check */ - ) { - char name[128]; - sprintf (name, "%s#%0*ld", - f->prefix_step_name, f->width_step_idx, (long)stepno); - return (H5Gget_info_by_name (f->file, name, NULL, H5P_DEFAULT) >= 0); -} - /*! \ingroup h5_core_filehandling diff --git a/src/h5core/h5_readwrite.c b/src/h5core/h5_readwrite.c index 6795c24..f13f8b3 100644 --- a/src/h5core/h5_readwrite.c +++ b/src/h5core/h5_readwrite.c @@ -200,7 +200,7 @@ h5priv_close_step ( static h5_err_t _set_step ( h5_file_t* const f, - const h5_int64_t step_idx /*!< [in] Step to set. */ + const h5_id_t step_idx /*!< [in] Step to set. */ ) { f->step_idx = step_idx; @@ -219,10 +219,10 @@ _set_step ( return H5_SUCCESS; } -h5_int64_t +h5_err_t h5_set_step ( - h5_file_t* const f, /*!< [in] Handle to open file */ - const h5_int64_t step_idx /*!< [in] Step to set. */ + h5_file_t* const f, /*!< [in] Handle to open file */ + const h5_id_t step_idx /*!< [in] Step to set. */ ) { TRY( h5priv_close_step (f) ); @@ -290,15 +290,15 @@ h5_get_dataset_type( } h5_err_t -h5_has_index ( +h5_has_step ( h5_file_t* const f, /*!< [in] Handle to open file */ - const h5_int64_t step /*!< [in] Step number to query */ + const h5_id_t step /*!< [in] Step number to query */ ) { char name[2*H5_STEPNAME_LEN]; sprintf (name, "%s#%0*lld", f->prefix_step_name, f->width_step_idx, (long long)step); - return (H5Gget_objinfo(f->file, name, 1, NULL) >= 0); + return h5priv_hdf5_link_exists(f, f->file, name); } h5_err_t @@ -309,8 +309,9 @@ h5_normalize_dataset_name ( ) { if ( strlen(name) > H5_DATANAME_LEN ) { - strncpy ( name2, name, H5_DATANAME_LEN - 1 ); + strncpy ( name2, name, H5_DATANAME_LEN-1 ); name2[H5_DATANAME_LEN-1] = '\0'; + h5_warn (f, "Truncated name '%s' to '%s'.", name, name2); } else { strcpy ( name2, name ); } @@ -318,8 +319,8 @@ h5_normalize_dataset_name ( if ( strcmp( name2, H5_BLOCKNAME ) == 0 ) { h5_error (f, H5_ERR_INVAL, - "Can't create dataset or field with name '%s' because it is " - "reserved by H5Block.", + "Can't create dataset or field with name '%s' because " + "it is reserved by H5Block.", H5_BLOCKNAME); } diff --git a/src/h5core/h5_types_private.h b/src/h5core/h5_types_private.h index d10cb09..36cf5d5 100644 --- a/src/h5core/h5_types_private.h +++ b/src/h5core/h5_types_private.h @@ -29,7 +29,7 @@ struct h5_file { hid_t xfer_prop; /* dataset transfer properties */ hid_t access_prop; /* file access properties */ hid_t create_prop; /* file create properties */ - hid_t root_gid; /* id of root group */ + hid_t root_gid; /* id of root */ hid_t step_gid; /* id of current step */ /* step internal data */ diff --git a/src/h5core/h5b_attribs.c b/src/h5core/h5b_attribs.c index dfab677..3597a28 100644 --- a/src/h5core/h5b_attribs.c +++ b/src/h5core/h5b_attribs.c @@ -11,7 +11,7 @@ h5_write_field_attrib ( const h5_int64_t attrib_nelem /*!< IN: number of elements */ ) { - TRY( h5bpriv_open_field_group(f, field_name) ); + TRY( h5bpriv_create_field_group(f, field_name) ); TRY( h5_write_attrib ( f, diff --git a/src/h5core/h5b_errorhandling_private.h b/src/h5core/h5b_errorhandling_private.h index d80fbbf..ab89fa3 100644 --- a/src/h5core/h5b_errorhandling_private.h +++ b/src/h5core/h5b_errorhandling_private.h @@ -6,12 +6,12 @@ return h5_error( \ f, \ H5_ERR_LAYOUT, \ - "No layout defined.") + "No view has been defined!") #define HANDLE_H5_LAYOUT_ERR( f ) \ h5_error( \ f, \ H5_ERR_LAYOUT, \ - "Bad layout."); + "Bad view!"); #endif diff --git a/src/h5core/h5b_model.c b/src/h5core/h5b_model.c index 90e290d..9ccc14e 100644 --- a/src/h5core/h5b_model.c +++ b/src/h5core/h5b_model.c @@ -17,75 +17,46 @@ static void _normalize_partition ( - struct h5b_partition *p /*!< IN/OUT: partition */ + h5b_partition_t *const p /*!< IN/OUT: partition */ ) { - h5_int64_t x; + h5_size_t tmp; if ( p->i_start > p->i_end ) { - x = p->i_start; + tmp = p->i_start; p->i_start = p->i_end; - p->i_end = x; + p->i_end = tmp; } if ( p->j_start > p->j_end ) { - x = p->j_start; + tmp = p->j_start; p->j_start = p->j_end; - p->j_end = x; + p->j_end = tmp; } if ( p->k_start > p->k_end ) { - x = p->k_start; + tmp = p->k_start; p->k_start = p->k_end; - p->k_end = x; + p->k_end = tmp; } } -/*! - \ingroup h5block_private - - \internal - - Gather layout to all processors - - \return H5_SUCCESS or error code -*/ -static h5_err_t -_allgather ( - h5_file_t *const f /*!< IN: file handle */ - ) { -#ifdef PARALLEL_IO - struct h5b_partition *partition = &f->b->user_layout[f->myproc]; - struct h5b_partition *layout = f->b->user_layout; - - TRY( h5priv_mpi_allgather(f, - partition, 1, f->b->partition_mpi_t, - layout, 1, f->b->partition_mpi_t, f->comm) ); -#endif - return H5_SUCCESS; -} - -/*! - \ingroup h5block_private - - \internal - - Get dimension sizes of block. These informations are stored inside the - block structure. -*/ +/* MLH: this could be improved with an MPI_Reduce and MAX operator... + * but the user_layout array-of-structs would need to be a struct-of-arrays */ static void -_get_dimension_sizes ( - h5_file_t *const f /*!< IN: file handle */ +_get_max_dimensions ( + h5_file_t *const f, + h5b_partition_t *const user_layout ) { int proc; - struct h5b_fdata *b = f->b; - struct h5b_partition *partition = b->user_layout; + h5b_fdata_t *b = f->b; + h5b_partition_t *p = user_layout; b->i_max = 0; b->j_max = 0; b->k_max = 0; - for ( proc = 0; proc < f->nprocs; proc++, partition++ ) { - if ( partition->i_end > b->i_max ) b->i_max = partition->i_end; - if ( partition->j_end > b->j_max ) b->j_max = partition->j_end; - if ( partition->k_end > b->k_max ) b->k_max = partition->k_end; + for ( proc = 0; proc < f->nprocs; proc++, p++ ) { + if ( p->i_end > b->i_max ) b->i_max = p->i_end; + if ( p->j_end > b->j_max ) b->j_max = p->j_end; + if ( p->k_end > b->k_max ) b->k_max = p->k_end; } } @@ -105,8 +76,8 @@ _get_dimension_sizes ( */ static int _have_ghostzone ( - const struct h5b_partition *p, /*!< IN: partition \c p */ - const struct h5b_partition *q /*!< IN: partition \c q */ + const h5b_partition_t *const p, /*!< IN: partition \c p */ + const h5b_partition_t *const q /*!< IN: partition \c q */ ) { return ( ! ( _NO_GHOSTZONE ( p, q ) || _NO_GHOSTZONE ( q, p ) ) ); } @@ -122,7 +93,7 @@ _have_ghostzone ( */ static h5_int64_t _volume_of_partition ( - const struct h5b_partition *p /*!< IN: partition */ + const h5b_partition_t *const p /*!< IN: partition */ ) { return (p->i_end - p->i_start) * (p->j_end - p->j_start) @@ -144,8 +115,8 @@ _volume_of_partition ( */ static h5_int64_t _volume_of_ghostzone ( - const struct h5b_partition *p, /*!< IN: ptr to first partition */ - const struct h5b_partition *q /*!< IN: ptr to second partition */ + const h5b_partition_t *const p, /*!< IN: ptr to first partition */ + const h5b_partition_t *const q /*!< IN: ptr to second partition */ ) { h5_int64_t dx = MIN ( p->i_end, q->i_end ) @@ -171,8 +142,8 @@ _volume_of_ghostzone ( */ static h5_int64_t _dissolve_X_ghostzone ( - struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */ - struct h5b_partition *q /*!< IN/OUT: ptr to second partition */ + h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */ + h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */ ) { if ( p->i_start > q->i_start ) @@ -199,8 +170,8 @@ _dissolve_X_ghostzone ( */ static h5_int64_t _dissolve_Y_ghostzone ( - struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */ - struct h5b_partition *q /*!< IN/OUT: ptr to second partition */ + h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */ + h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */ ) { if ( p->j_start > q->j_start ) @@ -227,8 +198,8 @@ _dissolve_Y_ghostzone ( */ static h5_int64_t _dissolve_Z_ghostzone ( - struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */ - struct h5b_partition *q /*!< IN/OUT: ptr to second partition */ + h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */ + h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */ ) { if ( p->k_start > q->k_start ) @@ -256,17 +227,17 @@ _dissolve_Z_ghostzone ( \return H5_SUCCESS or error code. */ -static h5_int64_t +static h5_err_t _dissolve_ghostzone ( h5_file_t *const f, - struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */ - struct h5b_partition *q /*!< IN/OUT: ptr to second partition */ + h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */ + h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */ ) { - struct h5b_partition p_; - struct h5b_partition q_; - struct h5b_partition p_best; - struct h5b_partition q_best; + h5b_partition_t p_; + h5b_partition_t q_; + h5b_partition_t p_best; + h5b_partition_t q_best; h5_int64_t vol; h5_int64_t max_vol = 0; @@ -338,36 +309,36 @@ _dissolve_ghostzone ( \return H5_SUCCESS or error code. */ -static h5_int64_t +static h5_err_t _dissolve_ghostzones ( - h5_file_t *const f /*!< IN: file handle */ + h5_file_t *const f, + const h5b_partition_t *const user_layout, + h5b_partition_t *const write_layout ) { - struct h5b_fdata *b = f->b; - struct h5b_partition *p; - struct h5b_partition *q; + h5b_partition_t *p; + h5b_partition_t *q; int proc_p, proc_q; struct list { struct list *prev; struct list *next; - struct h5b_partition *p; - struct h5b_partition *q; + h5b_partition_t *p; + h5b_partition_t *q; h5_int64_t vol; } *p_begin, *p_el, *p_max, *p_end, *p_save; - memcpy ( b->write_layout, b->user_layout, - f->nprocs * sizeof (*f->b->user_layout) ); + memcpy( write_layout, user_layout, f->nprocs*sizeof(h5b_partition_t) ); TRY( p_begin = (struct list*)h5priv_alloc(f, NULL, sizeof(*p_begin)) ); p_max = p_end = p_begin; - memset ( p_begin, 0, sizeof ( *p_begin ) ); + memset( p_begin, 0, sizeof ( *p_begin ) ); - for ( proc_p = 0, p = b->write_layout; + for ( proc_p = 0, p = write_layout; proc_p < f->nprocs-1; proc_p++, p++ ) { - for ( proc_q = proc_p+1, q = &b->write_layout[proc_q]; + for ( proc_q = proc_p+1, q = &write_layout[proc_q]; proc_q < f->nprocs; proc_q++, q++ ) { @@ -416,45 +387,6 @@ _dissolve_ghostzones ( } free ( p_begin ); - p = &b->user_layout[f->myproc]; - h5_debug (f, - "PROC[%d]: User layout: %lld:%lld, %lld:%lld, %lld:%lld", - f->myproc, - (long long)p->i_start, (long long)p->i_end, - (long long)p->j_start, (long long)p->j_end, - (long long)p->k_start, (long long)p->k_end ); - /* more detailed debug output: all procs report their view - of all other procs */ - for ( proc_p = 0, p = b->user_layout; - proc_p < f->nprocs; - proc_p++, p++ ) { - h5_debug (f, - "PROC[%d]: proc[%d]: User layout: %lld:%lld, %lld:%lld, %lld:%lld ", - f->myproc, proc_p, - (long long)p->i_start, (long long)p->i_end, - (long long)p->j_start, (long long)p->j_end, - (long long)p->k_start, (long long)p->k_end ); - } - - p = &b->write_layout[f->myproc]; - h5_debug (f, - "PROC[%d]: Ghost-zone layout: %lld:%lld, %lld:%lld, %lld:%lld", - f->myproc, - (long long)p->i_start, (long long)p->i_end, - (long long)p->j_start, (long long)p->j_end, - (long long)p->k_start, (long long)p->k_end ); - /* more detailed debug output: all procs report their view - of all other procs */ - for ( proc_p = 0, p = b->write_layout; - proc_p < f->nprocs; - proc_p++, p++ ) { - h5_debug (f, - "PROC[%d]: proc[%d]: Ghost-zone layout: %lld:%lld, %lld:%lld, %lld:%lld ", - f->myproc, proc_p, - (long long)p->i_start, (long long)p->i_end, - (long long)p->j_start, (long long)p->j_end, - (long long)p->k_start, (long long)p->k_end ); - } return H5_SUCCESS; } @@ -482,15 +414,31 @@ h5bpriv_open_block_group ( h5_file_t *const f /*!< IN: file handle */ ) { - struct h5b_fdata *b = f->b; + h5b_fdata_t *b = f->b; - if ( b->block_gid >= 0) { - TRY( h5priv_close_hdf5_group(f, b->block_gid) ); - b->block_gid = -1; - } + TRY( h5priv_close_hdf5_group(f, b->block_gid) ); + b->block_gid = h5priv_open_hdf5_group(f, f->step_gid, H5_BLOCKNAME); + if (f->b->block_gid < 0) + return h5_error(f, + H5_ERR_INVAL, + "Time step does not contain H5Block data!"); - if ( b->block_gid < 0 ) { - TRY( b->block_gid = h5priv_open_hdf5_group(f, + return H5_SUCCESS; +} + +static h5_err_t +_create_block_group ( + h5_file_t *const f /*!< IN: file handle */ + ) { + + h5_err_t exists; + TRY( exists = h5priv_hdf5_link_exists(f, f->step_gid, H5_BLOCKNAME) ); + + if (exists > 0) { + TRY( h5bpriv_open_block_group(f) ); + } else { + TRY( h5priv_close_hdf5_group(f, f->b->block_gid) ); + TRY( f->b->block_gid = h5priv_create_hdf5_group(f, f->step_gid, H5_BLOCKNAME) ); } @@ -506,10 +454,10 @@ h5bpriv_have_field_group ( char name2[H5_DATANAME_LEN]; h5_normalize_dataset_name(f, name, name2); - TRY( h5bpriv_open_block_group ( f ) ); + TRY( h5bpriv_open_block_group(f) ); h5_err_t exists; - TRY( exists = h5priv_hdf5_have_link(f, f->b->block_gid, name2) ); + TRY( exists = h5priv_hdf5_link_exists(f, f->b->block_gid, name2) ); return exists; } @@ -523,16 +471,44 @@ h5bpriv_open_field_group ( char name2[H5_DATANAME_LEN]; h5_normalize_dataset_name(f, name, name2); - if ( f->b->field_gid >= 0 ) { - TRY( h5priv_close_hdf5_group(f, f->b->field_gid) ); - } - - TRY( h5bpriv_open_block_group ( f ) ); - TRY( f->b->field_gid = h5priv_open_hdf5_group(f, f->b->block_gid, name2) ); + TRY( h5priv_close_hdf5_group(f, f->b->field_gid) ); + TRY( h5bpriv_open_block_group(f) ); + f->b->field_gid = h5priv_open_hdf5_group(f, f->b->block_gid, name2); + if (f->b->field_gid < 0) + return h5_error(f, + H5_ERR_INVAL, + "Field '%s' does not exist!", name2); return H5_SUCCESS; } +h5_err_t +h5bpriv_create_field_group ( + h5_file_t *const f, /*!< IN: file handle */ + const char *name /*!< IN: name of field group to create */ + ) { + + h5b_fdata_t *b = f->b; + + TRY( _create_block_group(f) ); + + char name2[H5_DATANAME_LEN]; + h5_normalize_dataset_name(f, name, name2); + + h5_err_t exists; + TRY( exists = h5priv_hdf5_link_exists(f, b->block_gid, name2) ); + + if (exists > 0) { + TRY( h5bpriv_open_field_group(f, name2) ); + } else { + TRY( h5priv_close_hdf5_group(f, f->b->field_gid) ); + TRY( b->field_gid = h5priv_create_hdf5_group(f, + b->block_gid, name2) ); + } + + return H5_SUCCESS; +} + h5_err_t h5b_3d_set_view ( h5_file_t *const f, /*!< IN: File handle */ @@ -544,22 +520,59 @@ h5b_3d_set_view ( const h5_size_t k_end /*!< IN: end index of \c k */ ) { - struct h5b_fdata *b = f->b; - struct h5b_partition *p = &b->user_layout[f->myproc]; + h5b_partition_t *p = f->b->user_layout; p->i_start = i_start; p->i_end = i_end; p->j_start = j_start; p->j_end = j_end; p->k_start = k_start; p->k_end = k_end; + _normalize_partition(p); - _normalize_partition ( p ); - TRY( _allgather( f ) ); - _get_dimension_sizes ( f ); - TRY( _dissolve_ghostzones ( f ) ); - TRY( _release_hyperslab ( f ) ); +#ifdef PARALLEL_IO + h5b_fdata_t *b = f->b; + h5b_partition_t *user_layout; + h5b_partition_t *write_layout; + + size_t size = f->nprocs * sizeof (h5b_partition_t); + TRY( user_layout = h5priv_alloc (f, NULL, size) ); + TRY( write_layout = h5priv_alloc (f, NULL, size) ); + + TRY( h5priv_mpi_allgather(f, + p, 1, f->b->partition_mpi_t, + user_layout, 1, f->b->partition_mpi_t, f->comm) ); + + _get_max_dimensions(f, user_layout); + + TRY( _dissolve_ghostzones(f, user_layout, write_layout) ); + b->user_layout[0] = user_layout[f->myproc]; + b->write_layout[0] = write_layout[f->myproc]; b->have_layout = 1; + p = b->user_layout; + h5_debug (f, + "[%d] User layout: %lld:%lld, %lld:%lld, %lld:%lld", + f->myproc, + (long long)p->i_start, (long long)p->i_end, + (long long)p->j_start, (long long)p->j_end, + (long long)p->k_start, (long long)p->k_end ); + + p = b->write_layout; + h5_debug (f, + "[%d] Ghost-zone layout: %lld:%lld, %lld:%lld, %lld:%lld", + f->myproc, + (long long)p->i_start, (long long)p->i_end, + (long long)p->j_start, (long long)p->j_end, + (long long)p->k_start, (long long)p->k_end ); + + + + free(user_layout); + free(write_layout); + + TRY( h5bpriv_release_hyperslab(f) ); +#endif + return H5_SUCCESS; } @@ -597,7 +610,7 @@ h5b_3d_get_chunk ( CHECK_TIMEGROUP ( f ); - struct h5b_fdata *b = f->b; + h5b_fdata_t *b = f->b; TRY( h5bpriv_open_field_group ( f, field_name ) ); @@ -639,7 +652,7 @@ h5b_3d_get_view ( if ( ( proc < 0 ) || ( proc >= f->nprocs ) ) return h5_error(f, H5_ERR_INVAL, "Invalid processor id %d!", proc); - struct h5b_partition *p = &f->b->user_layout[(size_t)proc]; + h5b_partition_t *p = &f->b->user_layout[(size_t)proc]; *i_start = p->i_start; *i_end = p->i_end; @@ -666,7 +679,7 @@ h5b_3d_get_reduced_view ( if ( ( proc < 0 ) || ( proc >= f->nprocs ) ) return h5_error(f, H5_ERR_INVAL, "Invalid processor id %d!", proc); - struct h5b_partition *p = &f->b->write_layout[(size_t)proc]; + h5b_partition_t *p = &f->b->write_layout[(size_t)proc]; *i_start = p->i_start; *i_end = p->i_end; @@ -686,7 +699,7 @@ h5b_3d_get_proc ( const h5_int64_t k /*!< IN: \c k coordinate */ ) { - struct h5b_partition *layout = f->b->write_layout; + h5b_partition_t *layout = f->b->write_layout; int proc; for ( proc = 0; proc < f->nprocs; proc++, layout++ ) { @@ -704,6 +717,8 @@ h5b_get_num_fields ( h5_file_t *const f /*!< IN: File handle */ ) { + CHECK_TIMEGROUP( f ); + TRY( h5bpriv_open_block_group(f) ); return h5priv_get_num_objs_in_hdf5_group( f, f->b->block_gid ); } @@ -712,14 +727,16 @@ h5_err_t h5b_get_field_info_by_name ( h5_file_t *const f, /*!< IN: file handle */ const char *name, /*!< OUT: field name */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ) { + CHECK_TIMEGROUP( f ); + hsize_t dims[16]; /* give it plenty of space even though we don't expect rank > 3 */ - hsize_t _grid_rank, _field_rank; + hsize_t _field_rank, _elem_rank; h5_size_t i, j; TRY( h5bpriv_open_field_group(f, name) ); @@ -731,18 +748,18 @@ h5b_get_field_info_by_name ( f->b->field_gid, H5_BLOCKNAME_X) ); TRY( dataspace_id = h5priv_get_hdf5_dataset_space(f, dataset_id) ); - TRY( _grid_rank = h5priv_get_dims_of_hdf5_dataspace(f, + TRY( _field_rank = h5priv_get_dims_of_hdf5_dataspace(f, dataspace_id, dims, NULL) ); - if ( grid_rank ) *grid_rank = (h5_size_t) _grid_rank; + if ( field_rank ) *field_rank = (h5_size_t) _field_rank; - if ( grid_dims ) { - for ( i = 0, j = _grid_rank-1; i < _grid_rank; i++, j-- ) - grid_dims[i] = (h5_size_t)dims[j]; + if ( field_dims ) { + for ( i = 0, j = _field_rank-1; i < _field_rank; i++, j-- ) + field_dims[i] = (h5_size_t)dims[j]; } - TRY( _field_rank = h5priv_get_num_objs_in_hdf5_group(f, - f->b->block_gid) ); - if ( field_rank ) *field_rank = (h5_size_t) _field_rank; + TRY( _elem_rank = h5priv_get_num_objs_in_hdf5_group(f, + f->b->field_gid) ); + if ( elem_rank ) *elem_rank = (h5_size_t) _elem_rank; hid_t h5type; TRY( h5type = h5priv_get_hdf5_dataset_type(f, dataset_id) ); @@ -762,12 +779,14 @@ h5b_get_field_info ( const h5_size_t idx, /*!< IN: index of field */ char *name, /*!< OUT: field name */ const h5_size_t len_name, /*!< IN: buffer size */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ) { + CHECK_TIMEGROUP( f ); + TRY( h5bpriv_open_block_group(f) ); TRY( h5priv_get_hdf5_objname_by_idx( f, @@ -777,6 +796,6 @@ h5b_get_field_info ( (size_t)len_name) ); return h5b_get_field_info_by_name(f, - name, grid_rank, grid_dims, field_rank, type); + name, field_rank, field_dims, elem_rank, type); } diff --git a/src/h5core/h5b_model_private.h b/src/h5core/h5b_model_private.h index e5f8c74..31c6a84 100644 --- a/src/h5core/h5b_model_private.h +++ b/src/h5core/h5b_model_private.h @@ -13,6 +13,12 @@ h5bpriv_open_field_group ( const char *name ); +h5_err_t +h5bpriv_create_field_group ( + h5_file_t *const f, /*!< IN: file handle */ + const char *name /*!< IN: name of field group to create */ + ); + h5_err_t h5bpriv_open_block_group ( h5_file_t *const f /*!< IN: file handle */ diff --git a/src/h5core/h5b_readwrite.c b/src/h5core/h5b_readwrite.c index 987bb23..aaf2e44 100644 --- a/src/h5core/h5b_readwrite.c +++ b/src/h5core/h5b_readwrite.c @@ -1,50 +1,6 @@ #include "h5core/h5_core.h" #include "h5_core_private.h" -static h5_err_t -_create_block_group ( - h5_file_t *const f /*!< IN: file handle */ - ) { - - h5_err_t exists; - TRY( exists = h5priv_hdf5_link_exists(f, f->step_gid, H5_BLOCKNAME) ); - - if (exists > 0) { - TRY( h5bpriv_open_block_group(f) ); - } else { - TRY( f->b->block_gid = h5priv_create_hdf5_group(f, - f->step_gid, H5_BLOCKNAME) ); - } - - return H5_SUCCESS; -} - -static h5_err_t -_create_field_group ( - h5_file_t *const f, /*!< IN: file handle */ - const char *name /*!< IN: name of field group to create */ - ) { - - struct h5b_fdata *b = f->b; - - TRY( _create_block_group(f) ); - - char name2[H5_DATANAME_LEN]; - h5_normalize_dataset_name(f, name, name2); - - h5_err_t exists; - TRY( exists = h5priv_hdf5_link_exists(f, b->block_gid, name2) ); - - if (exists > 0) { - TRY( h5bpriv_open_field_group(f, name2) ); - } else { - TRY( b->field_gid = h5priv_create_hdf5_group(f, - b->block_gid, name2) ); - } - - return H5_SUCCESS; -} - static h5_err_t _select_hyperslab_for_writing ( h5_file_t *const f /*!< IN: file handle */ @@ -55,9 +11,9 @@ _select_hyperslab_for_writing ( */ if ( f->b->shape >= 0 ) return H5_SUCCESS; - struct h5b_fdata *b = f->b; - struct h5b_partition *p = &b->write_layout[f->myproc]; - struct h5b_partition *q = &b->user_layout[f->myproc]; + h5b_fdata_t *b = f->b; + h5b_partition_t *p = b->write_layout; + h5b_partition_t *q = b->user_layout; int rank = 3; @@ -156,7 +112,7 @@ _write_data ( ) { hid_t dataset; - struct h5b_fdata *b = f->b; + h5b_fdata_t *b = f->b; h5_err_t exists; TRY( exists = h5priv_hdf5_link_exists (f, b->field_gid, data_name) ); @@ -203,7 +159,10 @@ h5b_write_scalar_data ( const void *data, /*!< IN: data to write */ const hid_t type /*!< IN: data type */ ) { - TRY( _create_field_group(f, field_name) ); + CHECK_TIMEGROUP( f ); + CHECK_WRITABLE_MODE( f ); + CHECK_LAYOUT( f ); + TRY( h5bpriv_create_field_group(f, field_name) ); TRY( _select_hyperslab_for_writing(f) ); TRY( _write_data(f, field_name, H5_BLOCKNAME_X, data, type) ); return H5_SUCCESS; @@ -218,7 +177,10 @@ h5b_write_vector3d_data ( const void *zdata, /*!< IN: z data to write */ const hid_t type /*!< IN: data type */ ) { - TRY( _create_field_group(f, field_name) ); + CHECK_TIMEGROUP( f ); + CHECK_WRITABLE_MODE( f ); + CHECK_LAYOUT( f ); + TRY( h5bpriv_create_field_group(f, field_name) ); TRY( _select_hyperslab_for_writing(f) ); TRY( _write_data(f, field_name, H5_BLOCKNAME_X, xdata, type) ); TRY( _write_data(f, field_name, H5_BLOCKNAME_Y, ydata, type) ); @@ -232,8 +194,8 @@ _select_hyperslab_for_reading ( const hid_t dataset ) { - struct h5b_fdata *b = f->b; - struct h5b_partition *p = &b->user_layout[f->myproc]; + h5b_fdata_t *b = f->b; + h5b_partition_t *p = b->user_layout; int rank; hsize_t field_dims[3]; hsize_t start[3] = { @@ -314,7 +276,7 @@ _read_data ( ) { hid_t dataset; - struct h5b_fdata *b = f->b; + h5b_fdata_t *b = f->b; TRY( dataset = h5priv_open_hdf5_dataset(f, b->field_gid, data_name) ); TRY( _select_hyperslab_for_reading(f, dataset) ); @@ -337,6 +299,8 @@ h5b_read_scalar_data ( void *data, /*!< OUT: read bufer */ const hid_t type /*!< IN: data type */ ) { + CHECK_TIMEGROUP( f ); + CHECK_LAYOUT( f ); TRY( h5bpriv_open_field_group(f, field_name) ); TRY( _read_data(f, field_name, H5_BLOCKNAME_X, data, type) ); return H5_SUCCESS; @@ -351,6 +315,8 @@ h5b_read_vector3d_data ( void *zdata, /*!< IN: z data to write */ const hid_t type /*!< IN: data type */ ) { + CHECK_TIMEGROUP( f ); + CHECK_LAYOUT( f ); TRY( h5bpriv_open_field_group(f, field_name) ); TRY( _read_data(f, field_name, H5_BLOCKNAME_X, xdata, type) ); TRY( _read_data(f, field_name, H5_BLOCKNAME_Y, ydata, type) ); diff --git a/src/h5core/h5b_types_private.h b/src/h5core/h5b_types_private.h index 3c9acce..5a41fe4 100644 --- a/src/h5core/h5b_types_private.h +++ b/src/h5core/h5b_types_private.h @@ -15,8 +15,8 @@ struct h5b_fdata { h5_size_t i_max; h5_size_t j_max; h5_size_t k_max; - struct h5b_partition* user_layout; - struct h5b_partition* write_layout; + struct h5b_partition user_layout[1]; + struct h5b_partition write_layout[1]; int have_layout; hid_t shape; @@ -29,4 +29,5 @@ struct h5b_fdata { MPI_Datatype partition_mpi_t; }; typedef struct h5b_fdata h5b_fdata_t; +typedef struct h5b_partition h5b_partition_t; #endif diff --git a/src/h5core/h5t_openclose.c b/src/h5core/h5t_openclose.c index b5e49b9..0b4690a 100644 --- a/src/h5core/h5t_openclose.c +++ b/src/h5core/h5t_openclose.c @@ -313,6 +313,18 @@ init_fdata ( t->dsinfo_num_elems_on_level.chunk_dims) ); t->dsinfo_num_elems_on_level.access_prop = H5P_DEFAULT; + /* initialize pointers */ + t->elems.data = NULL; + t->num_elems = NULL; + t->elems_ldta = NULL; + t->num_elems_on_level = NULL; + t->map_elem_g2l.items = NULL; + t->vertices = NULL; + t->vertices_data = NULL; + t->num_vertices = NULL; + t->map_vertex_g2l.items = NULL; + t->mtags.names = NULL; + return H5_SUCCESS; } @@ -348,25 +360,6 @@ h5tpriv_open_file ( return H5_SUCCESS; } -/*! - \ingroup h5_private - - \internal - - De-initialize topological internal structure. Open HDF5 objects are - closed and allocated memory freed. - - \return H5_SUCCESS or error code -*/ -h5_err_t -h5tpriv_close_file ( - h5_file_t* const f /*!< IN: file handle */ - ) { - TRY( h5t_close_mesh (f) ); - - return H5_SUCCESS; -} - h5_err_t h5tpriv_init_step ( h5_file_t* const f @@ -533,7 +526,9 @@ h5_err_t h5t_close_mesh ( h5_file_t* const f ) { - TRY( h5tpriv_write_mesh (f) ); + if (!(f->mode & H5_O_RDONLY)) { + TRY( h5tpriv_write_mesh (f) ); + } TRY( release_memory (f) ); TRY( init_fdata (f) ); @@ -575,3 +570,25 @@ h5tpriv_alloc_num_vertices ( return H5_SUCCESS; } + +/*! + \ingroup h5_private + + \internal + + De-initialize topological internal structure. Open HDF5 objects are + closed and allocated memory freed. + + \return H5_SUCCESS or error code +*/ +h5_err_t +h5tpriv_close_file ( + h5_file_t* const f /*!< IN: file handle */ + ) { + TRY( release_memory (f) ); + TRY( h5t_close_mesh (f) ); + + return H5_SUCCESS; +} + + diff --git a/src/h5core/h5u_model.c b/src/h5core/h5u_model.c index 1259fa4..5f22fba 100644 --- a/src/h5core/h5u_model.c +++ b/src/h5core/h5u_model.c @@ -1,45 +1,11 @@ #include "h5core/h5_core.h" #include "h5_core_private.h" -h5_int64_t +h5_ssize_t h5u_get_num_particles ( h5_file_t *const f /*!< [in] Handle to open file */ ) { h5_int64_t nparticles; - ssize_t exists; - - /* returns 0 if there are no datasets on disk */ - TRY ( exists = h5_get_num_hdf5_datasets(f, f->step_gid ) ); - if ( exists == 0 ) - { - /* try to recover number of particles from a previous - * H5PartSetNumParticles call. */ -#ifdef PARALLEL_IO - hsize_t total; - TRY( h5priv_mpi_sum(f, - &(f->u->nparticles), &total, - 1, MPI_LONG_LONG, f->comm) ); - nparticles = (h5_int64_t)total; -#else - nparticles = (h5_int64_t)f->u->nparticles; -#endif - if ( nparticles > 0 ) { - h5_debug ( - f, - "Using existing view to report " - "nparticles = %lld", (long long)nparticles ); - return nparticles; - } - else { - h5_warn ( - f, - "There are no datasets in the current timestep " - "nor existing views: " - "reporting 0 particles."); - return 0; - } - - } /* if a view exists, use its size as the number of particles */ if ( h5u_has_view ( f ) ) @@ -47,38 +13,52 @@ h5u_get_num_particles ( TRY( nparticles = h5priv_get_selected_npoints_of_hdf5_dataspace( f, f->u->diskshape) ); - h5_debug( - f, - "Found %lld points with H5Sget_select_npoints", - (long long)nparticles ); + h5_debug(f, + "Found %lld particles in existing view.", + (long long)nparticles ); } - /* otherwise, report all particles on disk in the first dataset - * for this timestep */ - else - { + else if ( f->u->shape > 0 ) { + TRY( nparticles = h5priv_get_npoints_of_hdf5_dataspace(f, f->u->shape) ); + h5_debug(f, + "Found %lld particles from previous H5PartSetNumParticles call.", + (long long)nparticles ); + } + else { + /* otherwise, report all particles on disk in the first dataset + * for this timestep */ char dataset_name[H5_DATANAME_LEN]; - TRY( h5priv_get_hdf5_objname_by_idx( + dataset_name[0] = '\0'; + h5_err_t exists = h5_get_hdf5_datasetname_by_idx( f, f->step_gid, 0, dataset_name, - H5_DATANAME_LEN) ); - TRY( nparticles = h5priv_get_npoints_of_hdf5_dataset_by_name ( + H5_DATANAME_LEN); + if ( exists < 0 ) + return h5_error(f, + H5_ERR_INVAL, + "Cannot determine the number of particles: " + "H5PartSetNumParticles has not been called, " + "no view has been set, and there are no " + "data sets for this time step!"); + TRY( nparticles = h5priv_get_npoints_of_hdf5_dataset_by_name( f, f->step_gid, - dataset_name ) ); + dataset_name) ); + h5_debug(f, + "Found %lld particles in the first data set of this time step.", + (long long)nparticles ); } return nparticles; } -h5_int64_t +h5_err_t h5u_set_num_particles ( h5_file_t *const f, /*!< [in] Handle to open file */ - const h5_int64_t nparticles, /*!< [in] Number of particles */ - const h5_int64_t stride /*!< [in] Stride of particles in memory */ + const h5_size_t nparticles, /*!< [in] Number of particles */ + const h5_size_t stride /*!< [in] Stride of particles in memory */ ) { - CHECK_FILEHANDLE( f ); struct h5u_fdata *u = f->u; hsize_t hstride; @@ -88,22 +68,14 @@ h5u_set_num_particles ( hsize_t dmax = H5S_UNLIMITED; if ( nparticles <= 0 ) - return h5_error( - f, + return h5_error(f, H5_ERR_INVAL, "Invalid number particles: %lld!\n", (long long)nparticles); - /* prevent invalid stride value */ - if (stride < 1) - { - h5_warn ( - f, - "Stride < 1 was specified: changing to 1." ); - hstride = 1; - } else { - hstride = (hsize_t)stride; - } + hstride = (hsize_t)stride; + if ( hstride > 1 ) + h5_debug( f, "Striding by %lld elements.", (long long)hstride); #ifndef PARALLEL_IO /* @@ -135,22 +107,16 @@ h5u_set_num_particles ( { start = 0; count = u->nparticles; - TRY( h5priv_select_hyperslab_of_hdf5_dataspace( - f, + TRY( h5priv_select_hyperslab_of_hdf5_dataspace(f, u->memshape, H5S_SELECT_SET, - &start, - &hstride, - &count, NULL ) ); + &start, &hstride, &count, + NULL) ); } #ifndef PARALLEL_IO count = u->nparticles; - TRY( u->shape = h5priv_create_hdf5_dataspace ( - f, - 1, - &count, - NULL ) ); + TRY( u->shape = h5priv_create_hdf5_dataspace(f, 1, &count, NULL) ); u->viewstart = 0; u->viewend = nparticles - 1; // view range is *inclusive* #else /* PARALLEL_IO */ @@ -173,30 +139,33 @@ h5u_set_num_particles ( &(u->nparticles), &total, 1, MPI_LONG_LONG, f->comm ) ); TRY( h5priv_mpi_prefix_sum(f, &(u->nparticles), &start, 1, MPI_LONG_LONG, f->comm ) ); + start -= u->nparticles; + + h5_debug(f, "Total particles across all processors: %lld.", + (long long)total); u->viewstart = start; u->viewend = start + u->nparticles - 1; // view range is *inclusive* /* declare overall datasize */ count = total; - TRY ( u->shape = h5priv_create_hdf5_dataspace (f, 1, &count, NULL) ); + TRY( u->shape = h5priv_create_hdf5_dataspace(f, 1, &count, NULL) ); /* declare overall data size but then will select a subset */ - TRY ( u->diskshape = h5priv_create_hdf5_dataspace (f, 1, &count, NULL) ); + TRY( u->diskshape = h5priv_create_hdf5_dataspace(f, 1, &count, NULL) ); count = nparticles; hstride = 1; - TRY ( h5priv_select_hyperslab_of_hdf5_dataspace ( - f, + TRY( h5priv_select_hyperslab_of_hdf5_dataspace(f, u->diskshape, H5S_SELECT_SET, &start, &hstride, &count, - NULL ) ); + NULL) ); #endif return H5_SUCCESS; } -h5_int64_t +h5_err_t h5u_has_view ( const h5_file_t *const f ) { @@ -264,8 +233,6 @@ h5u_set_view ( if ( start == -1 ) start = 0; if ( end == -1 ) end = total - 1; // range is *inclusive* - h5_debug ( f, "Total nparticles=%lld", (long long)total ); - /* so, is this selection inclusive or exclusive? it appears to be inclusive for both ends of the range. */ @@ -282,7 +249,9 @@ h5u_set_view ( u->viewend = end; u->nparticles = end - start + 1; - h5_debug ( f, "nparticles=%lld", (long long)u->nparticles ); + h5_debug (f, + "This view selected %lld particles.", + (long long)u->nparticles ); /* declare overall data size but then will select a subset */ TRY ( u->diskshape = h5priv_create_hdf5_dataspace ( f, 1, &total, NULL ) ); @@ -304,11 +273,11 @@ h5u_set_view ( return H5_SUCCESS; } -h5_int64_t +h5_err_t h5u_set_view_indices ( h5_file_t *const f, /*!< [in] Handle to open file */ - const h5_int64_t *const indices, /*!< [in] List of indices */ - const h5_int64_t nelems /*!< [in] Size of list */ + const h5_id_t *const indices, /*!< [in] List of indices */ + const h5_size_t nelems /*!< [in] Size of list */ ) { hsize_t total; @@ -338,19 +307,12 @@ h5u_set_view_indices ( return H5_SUCCESS; } - h5_debug ( f, "Total nparticles=%lld", (long long)total ); - if ( total == 0 ) return H5_SUCCESS; - /* check length of list */ - if ( nelems < 0 ) { - h5_warn (f, - "Array of view indices has length < 0: " - "resetting view."); - u->nparticles = 0; - } else { - u->nparticles = (hsize_t) nelems; - } + u->nparticles = (hsize_t) nelems; + h5_debug (f, + "This view selected %lld particles.", + (long long)u->nparticles ); /* declare overall data size but then will select a subset */ TRY ( u->diskshape = h5priv_create_hdf5_dataspace ( f, 1, &total, NULL ) ); @@ -371,7 +333,7 @@ h5u_set_view_indices ( return H5_SUCCESS; } -h5_int64_t +h5_err_t h5u_get_view ( h5_file_t *const f, h5_int64_t *start, @@ -444,7 +406,7 @@ h5u_set_canonical_view ( return H5_SUCCESS; } -h5_int64_t +h5_ssize_t h5u_get_num_datasets ( h5_file_t *const f /*!< [in] Handle to open file */ ) { @@ -456,15 +418,15 @@ h5u_get_num_datasets ( /*! Get information about dataset in current index given by its index */ -h5_int64_t +h5_err_t h5u_get_dataset_info ( h5_file_t *const f, /*!< [in] Handle to open file */ - const h5_int64_t idx, /*!< [in] Index of the dataset */ + const h5_id_t idx, /*!< [in] Index of the dataset */ char *dataset_name, /*!< [out] Name of dataset */ - const h5_int64_t len_dataset_name, + const h5_size_t len_dataset_name, /*!< [in] Size of buffer \c dataset_name */ h5_int64_t *type, /*!< [out] Type of data in dataset */ - h5_int64_t *nelem /*!< [out] Number of elements. */ + h5_size_t *nelem /*!< [out] Number of elements. */ ) { TRY( h5_get_hdf5_datasetname_by_idx ( diff --git a/src/h5core/h5u_readwrite.c b/src/h5core/h5u_readwrite.c index bd107ab..aa605bb 100644 --- a/src/h5core/h5u_readwrite.c +++ b/src/h5core/h5u_readwrite.c @@ -9,6 +9,8 @@ h5u_read_data ( const hid_t type ) { + CHECK_TIMEGROUP( f ); + struct h5u_fdata *u = f->u; hid_t dataset_id; hid_t space_id; @@ -107,9 +109,9 @@ h5u_write_data ( const hid_t type /*!< IN: Type of data */ ) { - CHECK_FILEHANDLE ( f ); - CHECK_WRITABLE_MODE( f ); CHECK_TIMEGROUP( f ); + CHECK_WRITABLE_MODE( f ); + struct h5u_fdata *u = f->u; hid_t dset_id; @@ -138,9 +140,6 @@ h5u_write_data ( H5P_DEFAULT) ); } -#ifdef PARALLEL_IO - TRY( h5_start_throttle(f) ); -#endif h5_info (f, "Writing dataset %s/%s.", h5_get_objname(f->step_gid), name2); @@ -153,9 +152,6 @@ h5u_write_data ( f->xfer_prop, data) ); TRY( h5priv_close_hdf5_dataset (f, dset_id) ); -#ifdef PARALLEL_IO - TRY( h5_end_throttle(f) ); -#endif f->empty = 0; diff --git a/src/include/H5.h b/src/include/H5.h index 187b6f6..3bf36f6 100644 --- a/src/include/H5.h +++ b/src/include/H5.h @@ -16,7 +16,6 @@ #ifndef __H5_H #define __H5_H -#include "H5_inquiry.h" #include "H5_attribs.h" h5_file_t * @@ -54,7 +53,7 @@ H5GetStepNameFormat ( h5_err_t H5SetStep ( h5_file_t *f, - const h5_int64_t step + const h5_id_t step ); h5_int64_t @@ -62,6 +61,22 @@ H5GetStep ( h5_file_t *f ); +int +H5GetNumProcs ( + h5_file_t * const f + ); + +h5_size_t +H5GetNumSteps ( + h5_file_t * const f + ); + +h5_err_t +H5HasStep ( + h5_file_t * const f, + h5_id_t step + ); + h5_err_t H5StartTraverseSteps ( h5_file_t *f diff --git a/src/include/H5Block.h b/src/include/H5Block.h index 89fef71..32d65f6 100644 --- a/src/include/H5Block.h +++ b/src/include/H5Block.h @@ -73,9 +73,9 @@ H5BlockGetFieldInfo ( const h5_size_t idx, /*!< IN: index of field */ char *name, /*!< OUT: field name */ const h5_size_t len_name, /*!< IN: buffer size */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ); @@ -83,9 +83,9 @@ h5_err_t H5BlockGetFieldInfoByName ( h5_file_t *const f, /*!< IN: file handle */ const char *name, /*!< IN: field name */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ - h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ); diff --git a/src/include/H5Part.h b/src/include/H5Part.h index 639a224..085f8ab 100644 --- a/src/include/H5Part.h +++ b/src/include/H5Part.h @@ -4,20 +4,20 @@ h5_err_t H5PartSetNumParticles ( h5_file_t *f, /*!< [in] Handle to open file */ - h5_int64_t nparticles /*!< [in] Number of particles */ + h5_size_t nparticles /*!< [in] Number of particles */ ); h5_err_t H5PartSetNumParticlesStrided ( h5_file_t *f, /*!< [in] Handle to open file */ - h5_int64_t nparticles, /*!< [in] Number of particles */ - h5_int64_t stride /*!< [in] Stride value (e.g. number of fields in the particle array) */ - ); + h5_size_t nparticles, /*!< [in] Number of particles */ + h5_size_t stride /*!< [in] Stride value (e.g. number of fields in the particle array) */ + ); h5_err_t -H5PartSetChunkSize ( +H5PartSetChunk ( h5_file_t *f, - h5_int64_t size + h5_size_t size ); h5_err_t @@ -76,31 +76,31 @@ H5PartReadDataInt32 ( h5_int32_t *data /*!< [out] Array of data */ ); -h5_int64_t +h5_ssize_t H5PartGetNumDatasets ( h5_file_t *f /*!< [in] Handle to open file */ ); -h5_int64_t +h5_err_t H5PartGetDatasetName ( h5_file_t *f, /*!< [in] Handle to open file */ - const h5_int64_t idx, /*!< [in] Index of the dataset */ + const h5_id_t idx, /*!< [in] Index of the dataset */ char *name, /*!< [out] Name of dataset */ - const h5_int64_t len /*!< [in] Size of buffer \c name */ + const h5_size_t len /*!< [in] Size of buffer \c name */ ); -h5_int64_t +h5_err_t H5PartGetDatasetInfo ( h5_file_t *f, /*!< [in] Handle to open file */ - const h5_int64_t idx,/*!< [in] Index of the dataset */ + const h5_id_t idx, /*!< [in] Index of the dataset */ char *dataset_name, /*!< [out] Name of dataset */ - const h5_int64_t len_dataset_name, + const h5_size_t len_dataset_name, /*!< [in] Size of buffer \c dataset_name */ h5_int64_t *type, /*!< [out] Type of data in dataset */ - h5_int64_t *nelem /*!< [out] Number of elements. */ + h5_size_t *nelem /*!< [out] Number of elements. */ ); -h5_int64_t +h5_ssize_t H5PartGetNumParticles ( h5_file_t *f /*!< [in] Handle to open file */ ); @@ -110,7 +110,7 @@ H5PartResetView ( h5_file_t *f /*!< [in] Handle to open file */ ); -h5_int64_t +h5_err_t H5PartHasView ( h5_file_t *f /*!< [in] Handle to open file */ ); @@ -125,8 +125,8 @@ H5PartSetView ( h5_err_t H5PartSetViewIndices ( h5_file_t *f, /*!< [in] Handle to open file */ - const h5_int64_t *indices, /*!< [in] List of indices */ - h5_int64_t nelems /*!< [in] Size of list */ + const h5_id_t *indices, /*!< [in] List of indices */ + h5_size_t nelems /*!< [in] Size of list */ ); h5_err_t @@ -146,4 +146,5 @@ H5PartSetCanonicalView ( h5_file_t *f /*!< [in] Handle to open file */ ); + #endif diff --git a/src/include/H5_inquiry.h b/src/include/H5_inquiry.h deleted file mode 100644 index 5d0afb0..0000000 --- a/src/include/H5_inquiry.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __H5_INQUIRY_H -#define __H5_INQUIRY_H - -int -H5GetNumNodes ( - h5_file_t * const f - ); - -h5_size_t -H5GetNumSteps ( - h5_file_t * const f - ); - -h5_err_t -H5HasStep ( - h5_file_t * const f, - h5_id_t step - ); - -#endif - diff --git a/src/include/h5core/h5_attribs.h b/src/include/h5core/h5_attribs.h index bf7c04d..5963e63 100644 --- a/src/include/h5core/h5_attribs.h +++ b/src/include/h5core/h5_attribs.h @@ -1,5 +1,5 @@ -#ifndef __H5_ATTRIBS_H -#define __H5_ATTRIBS_H +#ifndef __H5CORE_ATTRIBS_H +#define __H5CORE_ATTRIBS_H #define H5_ATTRIB_FILE 0 #define H5_ATTRIB_STEP 1 diff --git a/src/include/h5core/h5_errorhandling.h b/src/include/h5core/h5_errorhandling.h index 6441cec..210d660 100644 --- a/src/include/h5core/h5_errorhandling.h +++ b/src/include/h5core/h5_errorhandling.h @@ -26,7 +26,7 @@ return h5_error ( \ f, \ H5_ERR_INVAL, \ - "Internal error: step_gid <= 0."); + "Time step is invalid! Have you set the time step?"); #define h5_error_not_implemented( f, file, func, lino ) \ h5_error( \ diff --git a/src/include/h5core/h5_hdf5.h b/src/include/h5core/h5_hdf5.h index 3551179..ac359a4 100644 --- a/src/include/h5core/h5_hdf5.h +++ b/src/include/h5core/h5_hdf5.h @@ -20,7 +20,7 @@ h5_get_hdf5_groupname_by_idx ( hid_t loc_id, hsize_t idx, char *name, - size_t size + size_t len ); ssize_t @@ -35,7 +35,7 @@ h5_get_hdf5_datasetname_by_idx ( hid_t loc_id, hsize_t idx, char *name, - size_t size + size_t len ); const char * diff --git a/src/include/h5core/h5b_model.h b/src/include/h5core/h5b_model.h index 193e5cb..c9023ce 100644 --- a/src/include/h5core/h5b_model.h +++ b/src/include/h5core/h5b_model.h @@ -68,9 +68,9 @@ h5_err_t h5b_get_field_info_by_name ( h5_file_t *const f, /*!< IN: file handle */ const char *name, /*!< OUT: field name */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ); @@ -80,9 +80,9 @@ h5b_get_field_info ( const h5_size_t idx, /*!< IN: index of field */ char *name, /*!< OUT: field name */ const h5_size_t len_name, /*!< IN: buffer size */ - h5_size_t *grid_rank, /*!< OUT: grid rank */ - h5_size_t *grid_dims, /*!< OUT: grid dimensions */ h5_size_t *field_rank, /*!< OUT: field rank */ + h5_size_t *field_dims, /*!< OUT: field dimensions */ + h5_size_t *elem_rank, /*!< OUT: element rank */ h5_int64_t *type /*!< OUT: datatype */ ); diff --git a/src/include/h5core/h5u_model.h b/src/include/h5core/h5u_model.h index d792978..b14f198 100644 --- a/src/include/h5core/h5u_model.h +++ b/src/include/h5core/h5u_model.h @@ -1,67 +1,67 @@ #ifndef __H5U_MODEL_H #define __H5U_MODEL_H -h5_int64_t +h5_ssize_t h5u_get_num_particles ( h5_file_t *const f ); -h5_int64_t +h5_err_t h5u_set_num_particles ( h5_file_t *const f, - const h5_int64_t nparticles, - const h5_int64_t stride + const h5_size_t nparticles, + const h5_size_t stride ); -h5_int64_t +h5_err_t h5u_has_view ( const h5_file_t *const f ); -h5_int64_t +h5_err_t h5u_reset_view ( h5_file_t *const f ); -h5_int64_t +h5_err_t h5u_set_view ( h5_file_t *const f, h5_int64_t start, h5_int64_t end ); -h5_int64_t +h5_err_t h5u_set_view_indices ( h5_file_t *const f, - const h5_int64_t *const indices, - const h5_int64_t nelems + const h5_id_t *const indices, + const h5_size_t nelems ); -h5_int64_t +h5_err_t h5u_get_view ( h5_file_t *const f, h5_int64_t *start, h5_int64_t *end ); -h5_int64_t +h5_err_t h5u_set_canonical_view ( h5_file_t *const f ); -h5_int64_t +h5_ssize_t h5u_get_num_datasets ( h5_file_t *const f ); -h5_int64_t +h5_err_t h5u_get_dataset_info ( h5_file_t *const f, - const h5_int64_t idx, + const h5_id_t idx, char *dataset_name, - const h5_int64_t len_dataset_name, + const h5_size_t len_dataset_name, h5_int64_t *type, - h5_int64_t *nelem + h5_size_t *nelem ); h5_err_t diff --git a/test/Makefile.am b/test/Makefile.am index 5b3fad9..3722a32 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -4,18 +4,18 @@ OBJEXT=o -HDFLIB = -L@HDF5ROOT@/lib -lhdf5 -lz @SZLIB@ -H5LIB = -L../src/lib -lH5hutC -lH5core +HDFLIB = -L@HDF5ROOT@/lib -lhdf5 -lz @SZLIB@ -lm +H5LIB = -L../src/lib -lH5hutC -lH5hut LIBS = $(H5LIB) $(HDFLIB) @MPILIB@ INCLUDES = -I../src/include -I@HDF5ROOT@/include @MPIINC@ -noinst_PROGRAMS = h5u_test +noinst_PROGRAMS = h5u_test h5b_test TESTS_ENVIRONMENT = env LD_LIBRARY_PATH=@HDF5ROOT@/lib:$(LD_LIBRARY_PATH) -TESTS = h5u_test +TESTS = h5u_test h5b_test -EXTRA_PROGRAMS = h5u_test +EXTRA_PROGRAMS = h5u_test h5b_test h5u_test_SOURCES = \ h5u_test.c \ @@ -25,6 +25,22 @@ h5u_test_SOURCES = \ testframe.h \ params.h +h5u_test_DEPENDENCIES = \ + ../src/lib/libH5hut.a \ + ../src/lib/libH5hutC.a + +h5b_test_SOURCES = \ + h5b_test.c \ + h5b_write.c \ + h5b_read.c \ + testframe.c \ + testframe.h \ + params.h + +h5b_test_DEPENDENCIES = \ + ../src/lib/libH5hut.a \ + ../src/lib/libH5hutC.a + clean: clean-am rm -f *.h5 diff --git a/test/h5b_read.c b/test/h5b_read.c new file mode 100644 index 0000000..a865cd4 --- /dev/null +++ b/test/h5b_read.c @@ -0,0 +1,197 @@ +#include +#include +#include "testframe.h" +#include "params.h" + +static void +test_read_field_attribs( + h5_file_t *file, + const char *field_name, + int position) +{ + h5_err_t status; + char name[ATTR_NAME_SIZE]; + char str[ATTR_NAME_SIZE]; + h5_int32_t i32; + h5_int64_t i64; + h5_float32_t f32; + h5_float64_t f64; + + TEST("Reading field attributes"); + + i64 = H5BlockGetNumFieldAttribs(file, field_name); + VALUE(i64 % 5, 0, "file attribute count"); + + get_attr_name(name, "str", position); + status = H5BlockReadFieldAttribString( + file, field_name, name, str); + RETURN(status, H5_SUCCESS, "H5BlockReadFieldAttribString"); + SVALUE(str, ATTR_STR_VAL, "string attribute"); + + get_attr_name(name, "i32", position); + status = H5BlockReadFieldAttribInt32( + file, field_name, name, &i32); + RETURN(status, H5_SUCCESS, "H5BlockReadFieldAttribInt32"); + IVALUE(i32, ATTR_INT32_VAL, "int32 attribute"); + + get_attr_name(name, "i64", position); + status = H5BlockReadFieldAttribInt64( + file, field_name, name, &i64); + RETURN(status, H5_SUCCESS, "H5BlockReadFieldAttribInt64"); + IVALUE(i64, ATTR_INT64_VAL, "int64 attribute"); + + get_attr_name(name, "f32", position); + status = H5BlockReadFieldAttribFloat32( + file, field_name, name, &f32); + RETURN(status, H5_SUCCESS, "H5BlockReadFieldAttribFloat32"); + FVALUE(f32, ATTR_FLOAT_VAL, "float32 attribute"); + + get_attr_name(name, "f64", position); + status = H5BlockReadFieldAttribFloat64( + file, field_name, name, &f64); + RETURN(status, H5_SUCCESS, "H5BlockReadFieldAttribFloat64"); + FVALUE(f64, ATTR_FLOAT_VAL, "float64 attribute"); +} + +static void +test_read_data64(h5_file_t *file, int step) +{ + extern h5_size_t layout[6]; + + int i,t; + int rank, nprocs; + h5_err_t status; + h5_int64_t val, type[2]; + char name[4]; + h5_size_t field_rank[2], field_dims[6], elem_rank[2]; + + double *e; + double *ex,*ey,*ez; + h5_int64_t *id; + + const size_t nelems = + (layout[1] - layout[0] + 1) * + (layout[3] - layout[2] + 1) * + (layout[5] - layout[4] + 1); + + e=(double*)malloc(nelems*sizeof(double)); + ex=(double*)malloc(nelems*sizeof(double)); + ey=(double*)malloc(nelems*sizeof(double)); + ez=(double*)malloc(nelems*sizeof(double)); + id=(h5_int64_t*)malloc(nelems*sizeof(h5_int64_t)); + + TEST("Verifying dataset info"); + +#if PARALLEL_IO + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); +#else + nprocs = 1; + rank = 2; +#endif + + status = H5SetStep(file, step); + RETURN(status, H5_SUCCESS, "H5SetStep"); + + val = H5BlockGetNumFields(file); + IVALUE(val, 3, "field count"); + + for (i=0; i<3; i++) { + status = H5BlockGetFieldInfo( + file, i, name, 4, + field_rank, field_dims, elem_rank, type); + RETURN(status, H5_SUCCESS, "H5BlockGetFieldInfo"); + + status = H5BlockGetFieldInfoByName( + file, name, + field_rank+1, field_dims+3, elem_rank+1, type+1); + RETURN(status, H5_SUCCESS, "H5BlockGetFieldInfoByName"); + IVALUE(field_rank[0], field_rank[1], "field rank"); + IVALUE(field_dims[0], field_dims[3], "field dims x"); + IVALUE(field_dims[1], field_dims[4], "field dims y"); + IVALUE(field_dims[2], field_dims[5], "field dims z"); + IVALUE(elem_rank[0], elem_rank[1], "elem rank"); + IVALUE(type[0], type[1], "field type"); + + IVALUE(field_rank[0], 3, "field rank"); + IVALUE(field_dims[0], NBLOCKX, "field dims x"); + IVALUE(field_dims[1], NBLOCKY, "field dims y"); + IVALUE(field_dims[2], NBLOCKZ, "field dims z"); + if (i==1) { + CVALUE(name[0], 'e', "field name"); + IVALUE(elem_rank[0], 1, "elem rank"); + IVALUE(type[0], H5_FLOAT64_T, "field type"); + } else if (i==0) { + CVALUE(name[0], 'E', "field name"); + IVALUE(elem_rank[0], 3, "elem rank"); + IVALUE(type[1], H5_FLOAT64_T, "field type"); + } else if (i==2) { + CVALUE(name[0], 'i', "field name"); + IVALUE(elem_rank[0], 1, "elem rank"); + IVALUE(type[1], H5_INT64_T, "field type"); + } + } + +#if PARALLEL_IO + TEST("Setting throttle"); + status = H5SetThrottle(file, 3); + RETURN(status, H5_SUCCESS, "H5SetThrottle"); +#endif + + TEST("Reading 64-bit data"); + + for (t=step; t +#include +#include + +#include "testframe.h" +#include "params.h" + +/* global */ +h5_size_t layout[6]; + +/* from write.c */ +void h5b_test_write1(void); + +/* from read.c */ +void h5b_test_read1(void); + +static int +_nth_root_int_divisor (const int m, const int n) +{ + int i, root; + double p; + + p = 1.0 / (double)n; + root = (int) ceil ( pow ( (double)m, p ) ); + for (i=root; i<=m; i++) + { + if (m % i == 0) return i; + } + + return i; +} + +int main(int argc, char **argv) +{ + extern h5_size_t layout[6]; +#ifdef PARALLEL_IO + MPI_Init(&argc, &argv); + + int procs, rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &procs); + + if (procs > MAX_MPI_TASKS) { + fprintf(stderr, + "ERROR: please use <= %d MPI tasks for the test.\n", + MAX_MPI_TASKS); + exit(EXIT_FAILURE); + } + + /* make up a 3D layout */ + h5_size_t decomp[3]; + decomp[0] = _nth_root_int_divisor (procs, 3); + decomp[1] = _nth_root_int_divisor (procs / decomp[0], 2); + decomp[2] = procs / decomp[0] / decomp[1]; + + h5_size_t i,j,k; + k = rank % decomp[2]; + j = (rank / decomp[2]) % decomp[1]; + i = rank / (decomp[2] * decomp[1]); + + layout[0] = i*NBLOCKX; + layout[1] = (i+1)*NBLOCKX - 1; + layout[2] = j*NBLOCKY; + layout[3] = (j+1)*NBLOCKY - 1; + layout[4] = k*NBLOCKZ; + layout[5] = (k+1)*NBLOCKZ - 1; +#else // PARALLEL_IO + layout[0] = 0; + layout[1] = NBLOCKX - 1; + layout[2] = 0; + layout[3] = NBLOCKY - 1; + layout[4] = 0; + layout[5] = NBLOCKZ - 1; +#endif + + /* Initialize testing framework */ + TestInit(argv[0], NULL, NULL); + + /* Tests are generally arranged from least to most complexity... */ + AddTest("write1", h5b_test_write1, NULL, "Write 64-bit data", NULL); + AddTest("read1", h5b_test_read1, NULL, "Read 64-bit data", NULL); + + /* Display testing information */ + TestInfo(argv[0]); + + /* Parse command line arguments */ + TestParseCmdLine(argc, argv); + + H5SetVerbosityLevel(GetTestVerbosity()); + + /* Perform requested testing */ + PerformTests(); + + /* Display test summary, if requested */ + if (GetTestSummary()) + TestSummary(); + + /* Clean up test files, if allowed */ + //if (GetTestCleanup() && !getenv("HDF5_NOCLEANUP")) + // TestCleanup(); + +#ifdef PARALLEL_IO + TestPrintf ("reached end\n"); + fflush(stdout); + MPI_Finalize(); +#endif + return GetTestNumErrs(); +} + diff --git a/test/h5b_write.c b/test/h5b_write.c new file mode 100644 index 0000000..9ada3e9 --- /dev/null +++ b/test/h5b_write.c @@ -0,0 +1,131 @@ +#include +#include "testframe.h" +#include "params.h" + +static void +test_write_field_attribs( + h5_file_t *file, + const char *field_name, + int position) +{ + h5_err_t status; + char name[ATTR_NAME_SIZE]; + + TEST("Writing field attributes"); + + get_attr_name(name, "str", position); + status = H5BlockWriteFieldAttribString( + file, field_name, name, ATTR_STR_VAL); + RETURN(status, H5_SUCCESS, "H5BlockWriteFieldAttribString"); + + get_attr_name(name, "i32", position); + h5_int32_t i32 = ATTR_INT32_VAL; + status = H5BlockWriteFieldAttribInt32( + file, field_name, name, &i32, 1); + RETURN(status, H5_SUCCESS, "H5BlockWriteFieldAttribInt32"); + + get_attr_name(name, "i64", position); + h5_int64_t i64 = ATTR_INT64_VAL; + status = H5BlockWriteFieldAttribInt64( + file, field_name, name, &i64, 1); + RETURN(status, H5_SUCCESS, "H5BlockWriteFieldAttribInt64"); + + get_attr_name(name, "f32", position); + h5_float32_t f32 = ATTR_FLOAT_VAL; + status = H5BlockWriteFieldAttribFloat32( + file, field_name, name, &f32, 1); + RETURN(status, H5_SUCCESS, "H5BlockWriteFieldAttribFloat32"); + + get_attr_name(name, "f64", position); + h5_float64_t f64 = ATTR_FLOAT_VAL; + status = H5BlockWriteFieldAttribFloat64( + file, field_name, name, &f64, 1); + RETURN(status, H5_SUCCESS, "H5BlockWriteFieldAttribFloat64"); +} + +static void +test_write_data64(h5_file_t *file, int step) +{ + extern h5_size_t layout[6]; + + int i,t; + h5_int64_t status, val; + + double *e; + double *ex,*ey,*ez; + h5_int64_t *id; + + const size_t nelems = + (layout[1] - layout[0] + 1) * + (layout[3] - layout[2] + 1) * + (layout[5] - layout[4] + 1); + + e=(double*)malloc(nelems*sizeof(double)); + ex=(double*)malloc(nelems*sizeof(double)); + ey=(double*)malloc(nelems*sizeof(double)); + ez=(double*)malloc(nelems*sizeof(double)); + id=(h5_int64_t*)malloc(nelems*sizeof(h5_int64_t)); + +#if PARALLEL_IO + TEST("Setting throttle"); + status = H5SetThrottle(file, 2); + RETURN(status, H5_SUCCESS, "H5SetThrottle"); +#endif + + TEST("Writing 64-bit data"); + + for (t=step; t