From 8dac86872be006981cf6207feac8f0de080aa5bd Mon Sep 17 00:00:00 2001 From: Marc Howison Date: Sun, 30 Aug 2009 18:02:04 +0000 Subject: [PATCH] added a proc shift option to H5MultiBlock, used in I/O benchmarking to defeat caching --- src/H5MultiBlock.c | 38 ++++++++++++++++++++++++++++++++++---- src/H5MultiBlock.h | 16 +++++++++++----- src/H5MultiBlockTypes.h | 1 + src/H5Part.c | 24 +++++++++--------------- src/H5Part.h | 7 ------- src/H5PartF.c | 2 -- src/H5PartPrivate.h | 4 ++-- 7 files changed, 57 insertions(+), 35 deletions(-) diff --git a/src/H5MultiBlock.c b/src/H5MultiBlock.c index 19853b7..8f2004e 100644 --- a/src/H5MultiBlock.c +++ b/src/H5MultiBlock.c @@ -124,9 +124,10 @@ _get_decomp_and_offsets ( mb->decomp[1], mb->decomp[0] ); - k = f->myproc % mb->decomp[2]; - j = (f->myproc / mb->decomp[2]) % mb->decomp[1]; - i = f->myproc / (mb->decomp[2] * mb->decomp[1]); + int rank = (f->myproc + mb->proc_shift) % f->nprocs; + k = rank % mb->decomp[2]; + j = (rank / mb->decomp[2]) % mb->decomp[1]; + i = rank / (mb->decomp[2] * mb->decomp[1]); /* keep track of blocks that border the edges of the field */ if (i == 0) mb->field_edges |= H5MB_EDGE_Z0; @@ -761,6 +762,7 @@ _H5MultiBlock_init ( mb->halo = 0; mb->read = 0; mb->have_decomp = 0; + mb->proc_shift = 0; return H5PART_SUCCESS; } @@ -1256,5 +1258,33 @@ H5MultiBlockFree ( return H5PART_SUCCESS; } -#endif +/*! + \ingroup h5multiblock_c_api + + Shifts the assignment of procs to blocks within the field. + + This is useful mainly for I/O benchmarking, in order to defeat the write + cache when reading in data that was just written out. + + \return \c H5PART_SUCCESS +*/ +h5part_int64_t +H5MultiBlockShiftProcs ( + H5PartFile *f, /* IN: file handle */ + const int shift /* IN: shift amount (non-negative) */ + ) { + + SET_FNAME ( "H5MultiBlockShiftProcs" ); + MULTIBLOCK_INIT ( f ); + + if ( shift < 0 ) return H5PART_ERR_INVAL; + + struct H5MultiBlockStruct *mb = f->multiblock; + + mb->proc_shift = shift; + + return H5PART_SUCCESS; +} + +#endif // PARALLEL_IO diff --git a/src/H5MultiBlock.h b/src/H5MultiBlock.h index e0fd0b1..127cc4a 100644 --- a/src/H5MultiBlock.h +++ b/src/H5MultiBlock.h @@ -29,10 +29,10 @@ H5MultiBlock3dDefineRadii ( h5part_int64_t H5MultiBlock3dDefineDims ( - H5PartFile *f, - const h5part_int64_t *field_dims, - const h5part_int64_t *block_dims - ); + H5PartFile *f, + const h5part_int64_t *field_dims, + const h5part_int64_t *block_dims + ); h5part_int64_t H5MultiBlock3dGetFieldDims( @@ -43,7 +43,7 @@ H5MultiBlock3dGetFieldDims( h5part_int64_t H5MultiBlock3dGetBlockDims( H5PartFile *f, - const char *field_name, + const char *field_name, h5part_int64_t *dims ); @@ -65,6 +65,12 @@ H5MultiBlockFree ( void *block ); +h5part_int64_t +H5MultiBlockShiftProcs ( + H5PartFile *f, + const int shift + ); + #ifdef __cplusplus } #endif diff --git a/src/H5MultiBlockTypes.h b/src/H5MultiBlockTypes.h index 4e28b4f..37305f5 100644 --- a/src/H5MultiBlockTypes.h +++ b/src/H5MultiBlockTypes.h @@ -12,6 +12,7 @@ struct H5MultiBlockStruct { int read; int halo; int have_decomp; + int proc_shift; }; #define H5PART_ERR_DECOMP -102 diff --git a/src/H5Part.c b/src/H5Part.c index e83343b..1854179 100644 --- a/src/H5Part.c +++ b/src/H5Part.c @@ -169,7 +169,6 @@ _H5Part_open_file ( if ( f_parallel ) { #ifdef PARALLEL_IO MPI_Info info = MPI_INFO_NULL; - MPI_Info_create(&info); if (MPI_Comm_size (comm, &f->nprocs) != MPI_SUCCESS) { HANDLE_MPI_COMM_SIZE_ERR; @@ -187,14 +186,6 @@ _H5Part_open_file ( goto error_cleanup; } - /* set IBM_largeblock_io for GPFS filesystems */ - if (flags & H5PART_FS_GPFS) { - if (f->myproc == 0) { - _H5Part_print_info ( "Setting IBM_largeblock_io hint." ); - } - MPI_Info_set ( info, "IBM_largeblock_io", "true" ); - } - /* select the HDF5 VFD */ if (flags & H5PART_VFD_MPIPOSIX) { if (f->myproc == 0) { @@ -237,6 +228,7 @@ _H5Part_open_file ( } f->create_prop = H5Pcreate(H5P_FILE_CREATE); H5Pset_istore_k (f->create_prop, H5PART_BTREE_IK); +#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8 /* defer metadata cache flushing until file close */ H5AC_cache_config_t cache_config; cache_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; @@ -248,10 +240,12 @@ _H5Part_open_file ( cache_config.flash_incr_mode = H5C_flash_incr__off; cache_config.decr_mode = H5C_decr__off; H5Pset_mdc_config (f->access_prop, &cache_config); +#elif + _H5Part_print_info ( + "Unable to defer metadata write: need HDF5 1.8"); +#endif } - MPI_Info_free(&info); - f->comm = comm; #endif } else { @@ -383,7 +377,7 @@ H5PartOpenFileParallel ( SET_FNAME ( "H5PartOpenFileParallel" ); int f_parallel = 1; /* parallel i/o */ - int align = 0; /* no tuning parameters */ + h5part_int64_t align = 0; /* no alignment tuning */ return _H5Part_open_file ( filename, flags, comm, f_parallel, align ); } @@ -404,7 +398,7 @@ H5PartOpenFileParallelAlign ( h5part_int64_t align /*!< [in] Alignment size in bytes. */ ) { - SET_FNAME ( "H5PartOpenFileParallel" ); + SET_FNAME ( "H5PartOpenFileParallelAlign" ); int f_parallel = 1; /* parallel i/o */ @@ -3111,12 +3105,12 @@ _H5Part_print_debug_detail ( void _H5Part_set_funcname ( - char * const fname + char *fname ) { __funcname = fname; } -char * const +char* _H5Part_get_funcname ( void ) { diff --git a/src/H5Part.h b/src/H5Part.h index 9fb6ed0..ce65b1a 100644 --- a/src/H5Part.h +++ b/src/H5Part.h @@ -9,12 +9,6 @@ extern "C" { #endif -#ifdef MPI_INCLUDED -#ifndef PARALLEL_IO -#define PARALLEL_IO -#endif -#endif - #include "H5PartTypes.h" #include "H5Block.h" @@ -40,7 +34,6 @@ extern "C" { #define H5PART_APPEND 0x04 #define H5PART_VFD_MPIPOSIX 0x08 #define H5PART_FS_LUSTRE 0x10 -#define H5PART_FS_GPFS 0x20 /* verbosity level flags */ #define H5PART_VERB_NONE 0 diff --git a/src/H5PartF.c b/src/H5PartF.c index c2264a4..9d885cf 100755 --- a/src/H5PartF.c +++ b/src/H5PartF.c @@ -226,8 +226,6 @@ _H5Part_flagsfor2c ( fbits |= H5PART_VFD_MPIPOSIX; else if ( strcmp ( flags, "fs_lustre" ) == 0 ) fbits |= H5PART_FS_LUSTRE; - else if (strcmp ( flags, "fs_gpfs" ) == 0 ) - fbits |= H5PART_FS_GPFS; flags = strtok ( NULL, "," ); } diff --git a/src/H5PartPrivate.h b/src/H5PartPrivate.h index 7e7cc5e..2beeef5 100644 --- a/src/H5PartPrivate.h +++ b/src/H5PartPrivate.h @@ -44,10 +44,10 @@ _H5Part_iteration_operator ( void _H5Part_set_funcname ( - char * const fname + char *fname ); -char * const +char* _H5Part_get_funcname ( void );