diff --git a/.gitattributes b/.gitattributes index 8198848..5d50cee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,6 +20,7 @@ doc/H5X_File_Format.txt -text doc/Makefile.am -text doc/doxyfooter -text examples/stridedf.F90 -text +examples/write_setview.c -text /install-sh -text /license.txt -text /missing -text diff --git a/Makefile.am b/Makefile.am index 8464cba..d63665e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,6 +7,8 @@ SUBDIRS = \ test \ tools +EXTRA_DIST = configure-crayxt + MAINTAINERCLEANFILES = \ config.h \ config.log \ diff --git a/NEWS b/NEWS index 83ee473..a7a97ed 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,25 @@ +#### H5PART 1.6.1 ############################################################ + +All Steps Available on Write +---------------------------- + +Previously, existing steps in a file were only accessable in read-only mode. +Now, all steps are available in all modes, including write-truncate and +write-append. Thus, it is now possible to overwrite existing data in write +mode. To help alert the user to this possibility, a warning is issued every +time an existing dataset is written to. Warnings can be printed by setting +the verbosity to H5PART_VERB_WARN or higher. + +New Throttling Routine +---------------------- + +Previously, a throttle factor of N meant that for P processors, the total +number of P writes were divided into N batchs of P/N writes. + +To better accomodate round-robin lustre striping, the new policy is to +execute P/N batch of N writes. Thus, matching N to the number of stripes +results in the desirable 1-1 matching of writers to stripes. + #### H5PART 1.6 ############################################################## Updated Documentation diff --git a/configure.ac b/configure.ac index e3c0593..4db535a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ # Every configure script must call AC_INIT before doing anything else. # AC_INIT (package, version, [bug-report], [tarname]) -AC_INIT([H5Part], [1.6.0], [h5part@lists.psi.ch], H5Part) +AC_INIT([H5Part], [1.6.1], [h5part@lists.psi.ch], H5Part) # Ensure that a recent enough version of Autoconf is being used. diff --git a/doc/Doxyfile b/doc/Doxyfile index 9c5f1ac..5309b38 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -4,7 +4,7 @@ # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = H5Part -PROJECT_NUMBER = 1.6.0 +PROJECT_NUMBER = 1.6.1 OUTPUT_DIRECTORY = ./ReferencePages CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/examples/write_setview.c b/examples/write_setview.c new file mode 100644 index 0000000..92c7e7b --- /dev/null +++ b/examples/write_setview.c @@ -0,0 +1,57 @@ +#include +#include +#include +#include + +#define DATASIZE 32 +#define ITERS 4 + +int main(int argc, char** argv) +{ + int i, rank, nprocs; + h5part_int32_t data[ITERS*DATASIZE]; + h5part_int64_t stat; + h5part_int64_t offset; + H5PartFile *file; + + // initialize MPI + MPI_Init (&argc, &argv); + MPI_Comm_rank (MPI_COMM_WORLD, &rank); + MPI_Comm_size (MPI_COMM_WORLD, &nprocs); + + H5PartSetVerbosityLevel(H5PART_VERB_DEBUG); + + file = H5PartOpenFileParallel("test.h5", H5PART_WRITE, MPI_COMM_WORLD); + assert (file != NULL); + + stat = H5PartSetStep(file, 0); + assert (stat == H5PART_SUCCESS); + + stat = H5PartSetNumParticles(file, ITERS*DATASIZE); + assert (stat == H5PART_SUCCESS); + + // create fake data + for (i=0; itimestep ); +#ifdef PARALLEL_IO + herr = _H5Part_end_throttle ( f ); + if ( herr < 0 ) return herr; +#endif + herr = H5Dclose ( dataset_id ); if ( herr < 0 ) return HANDLE_H5D_CLOSE_ERR; @@ -1448,6 +1458,11 @@ _H5Block_write_data ( ); if ( dataset < 0 ) return HANDLE_H5D_CREATE_ERR ( name, f->timestep ); +#ifdef PARALLEL_IO + herr = _H5Part_start_throttle ( f ); + if ( herr < 0 ) return herr; +#endif + herr = H5Dwrite ( dataset, type, @@ -1457,6 +1472,11 @@ _H5Block_write_data ( data ); if ( herr < 0 ) return HANDLE_H5D_WRITE_ERR ( name, f->timestep ); +#ifdef PARALLEL_IO + herr = _H5Part_end_throttle ( f ); + if ( herr < 0 ) return herr; +#endif + herr = H5Dclose ( dataset ); if ( herr < 0 ) return HANDLE_H5D_CLOSE_ERR; diff --git a/src/H5Part.c b/src/H5Part.c index 4e73a63..55c94ee 100644 --- a/src/H5Part.c +++ b/src/H5Part.c @@ -242,7 +242,7 @@ _H5Part_open_file ( } f->create_prop = H5Pcreate(H5P_FILE_CREATE); - H5Pset_istore_k (f->create_prop, H5PART_BTREE_IK); + H5Pset_istore_k (f->create_prop, btree_ik); #ifndef H5_USE_16_API /* defer metadata cache flushing until file close */ @@ -1851,9 +1851,11 @@ _H5Part_set_step ( char stepname[H5PART_STEPNAME_LEN]; _H5Part_get_step_name(f, step, stepname); +#if H5PART_SET_STEP_READ_ONLY if ( (!(f->flags & H5PART_READ)) && _H5Part_have_group ( f->file, stepname ) ) { return HANDLE_H5PART_STEP_EXISTS_ERR ( step ); } +#endif if ( f->timegroup >= 0 ) { herr_t herr = H5Gclose ( f->timegroup ); @@ -1862,6 +1864,9 @@ _H5Part_set_step ( f->timegroup = -1; f->timestep = step; +#if H5PART_SET_STEP_READ_ONLY + // in this mode, existing steps can be selecting only + // for a READ file handle if ( f->flags & H5PART_READ ) { _H5Part_print_debug ( "Proc[%d]: Set step to #%lld for file %lld", @@ -1893,6 +1898,33 @@ _H5Part_set_step ( return HANDLE_H5G_CREATE_ERR ( stepname ); } +#else // H5PART_SET_STEP_READ_ONLY + + // in this mode, existing steps can be selected for all file + // handles: first try to open the step, and create it if it + // doesn't exist + H5E_BEGIN_TRY + f->timegroup = H5Gopen ( f->file, stepname +#ifndef H5_USE_16_API + , H5P_DEFAULT +#endif + ); + H5E_END_TRY + + + if ( f->timegroup < 0 ) + { + f->timegroup = H5Gcreate( f->file, stepname, 0 +#ifndef H5_USE_16_API + , H5P_DEFAULT, H5P_DEFAULT +#endif + ); + if ( f->timegroup < 0 ) + return HANDLE_H5G_CREATE_ERR ( stepname ); + } + +#endif // H5PART_SET_STEP_READ_ONLY + return H5PART_SUCCESS; } @@ -3343,12 +3375,14 @@ H5PartSetThrottle ( if ( f->flags & H5PART_VFD_MPIIO_IND || f->flags & H5PART_VFD_MPIPOSIX ) { f->throttle = factor; - _H5Part_print_info ( - "Throttling set with factor '%d'", f->throttle ); + if ( f->myproc == 0 ) + _H5Part_print_info ( + "Throttling set with factor '%d'", f->throttle ); } else { - _H5Part_print_warn ( - "Throttling is only permitted with the MPI-POSIX " - "or MPI-IO Independent VFD." ); + if ( f->myproc == 0 ) + _H5Part_print_warn ( + "Throttling is only permitted with the MPI-POSIX " + "or MPI-IO Independent VFD." ); } return H5PART_SUCCESS; @@ -3366,14 +3400,14 @@ _H5Part_start_throttle ( _H5Part_print_info ("Throttling with factor = %d", f->throttle); } - if (f->myproc % f->throttle > 0) { + if (f->myproc / f->throttle > 0) { _H5Part_print_debug_detail ( "[%d] throttle: waiting on token from %d", - f->myproc, f->myproc - 1); + f->myproc, f->myproc - f->throttle); // wait to receive token before continuing with read ret = MPI_Recv( &token, 1, MPI_INT, - f->myproc - 1, // receive from previous proc + f->myproc - f->throttle, // receive from previous proc f->myproc, // use this proc id as message tag f->comm, MPI_STATUS_IGNORE @@ -3393,19 +3427,17 @@ _H5Part_end_throttle ( if (f->throttle > 0) { int ret; int token; - if (f->myproc % f->throttle < f->throttle - 1) { + if (f->myproc + f->throttle < f->nprocs) { // pass token to next proc - if (f->myproc + 1 < f->nprocs) { - _H5Part_print_debug_detail ( - "[%d] throttle: passing token to %d", - f->myproc, f->myproc + 1); - ret = MPI_Send( - &token, 1, MPI_INT, - f->myproc + 1, // send to next proc - f->myproc + 1, // use the id of the target as tag - f->comm - ); - } + _H5Part_print_debug_detail ( + "[%d] throttle: passing token to %d", + f->myproc, f->myproc + f->throttle); + ret = MPI_Send( + &token, 1, MPI_INT, + f->myproc + f->throttle, // send to next proc + f->myproc + f->throttle, // use the id of the target as tag + f->comm + ); if ( ret != MPI_SUCCESS ) return HANDLE_MPI_SENDRECV_ERR; } } diff --git a/src/H5Part.h b/src/H5Part.h index 05beebc..0baa461 100644 --- a/src/H5Part.h +++ b/src/H5Part.h @@ -16,10 +16,10 @@ extern "C" { #include "H5MultiBlock.h" #endif -#define H5PART_VER_STRING "1.6.0" +#define H5PART_VER_STRING "1.6.1" #define H5PART_VER_MAJOR 1 #define H5PART_VER_MINOR 6 -#define H5PART_VER_RELEASE 0 +#define H5PART_VER_RELEASE 1 /* error values */ #define H5PART_SUCCESS 0 diff --git a/src/H5PartPrivate.h b/src/H5PartPrivate.h index 1328154..d820de5 100644 --- a/src/H5PartPrivate.h +++ b/src/H5PartPrivate.h @@ -7,7 +7,7 @@ #define H5PART_GROUPNAME_STEP "Step" -#define H5PART_BTREE_IK 10000 +#define H5PART_SET_STEP_READ_ONLY 0 h5part_int64_t _H5Part_file_is_valid ( diff --git a/test/read.c b/test/read.c index 0391060..245ae4e 100644 --- a/test/read.c +++ b/test/read.c @@ -509,7 +509,7 @@ void test_read4(void) status = H5PartSetStep(file2, NTIMESTEPS); RETURN(status, H5PART_SUCCESS, "H5PartSetStep"); - test_read_data64(file2, NPARTICLES, NTIMESTEPS); + test_read_data64(file2, NPARTICLES, NTIMESTEPS-2); status = H5PartCloseFile(file1); RETURN(status, H5PART_SUCCESS, "H5PartCloseFile"); diff --git a/test/write.c b/test/write.c index e596459..c68eb28 100644 --- a/test/write.c +++ b/test/write.c @@ -65,7 +65,7 @@ static void test_write_data64(H5PartFile *file, int nparticles, int step) { int i,t; - h5part_int64_t status; + h5part_int64_t status, val; double *x,*y,*z; double *px,*py,*pz; @@ -108,10 +108,12 @@ test_write_data64(H5PartFile *file, int nparticles, int step) id[i] = i + nparticles*t; } + val = H5PartHasStep(file, t); + status = H5PartSetStep(file, t); RETURN(status, H5PART_SUCCESS, "H5PartSetStep"); - test_write_step_attribs(file, t); + if (val == 0) test_write_step_attribs(file, t); status = H5PartSetNumParticles(file, nparticles); RETURN(status, H5PART_SUCCESS, "H5PartSetNumParticles"); @@ -450,7 +452,7 @@ void test_write4(void) status = H5PartDefineStepName(file2, LONGNAME, 16); RETURN(status, H5PART_SUCCESS, "H5PartDefineStepName"); - test_write_data64(file1, NPARTICLES, NTIMESTEPS); + test_write_data64(file1, NPARTICLES, NTIMESTEPS-2); test_write_file_attribs(file1, 1); status = H5PartCloseFile(file1);