H5hut/src_old
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrated H5Part regression test and fixed numerous errors that it discovered.

Browse Source 
Finished integrating H5Block and the Fortran interface (untested).
Started adding an H5Block regression test with a few simple tests (all pass).
Added automatic detection of stripe information on lustre, and config option to compile against lustre API.
Moved buffers for H5Block ghost zone disolving out of the file handle and into the h5b_3d_set_view function.
Fixed bug with pointers in the H5Fed file data not being initialized to NULL.
This commit is contained in:
Marc Howison
2010-07-15 19:44:48 +00:00
parent 2c267f173a
commit 7ea00c38f8
50 changed files with 1404 additions and 747 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/h5core/Makefile.am
+9 -7
View File
@@ -38,6 +38,7 @@ EXTRA_HEADERS = \
h5_qsort_private.h \
h5_readwrite_private.h \
h5_syscall_private.h \
h5_lustre_private.h \
h5u_errorhandling_private.h \
h5b_errorhandling_private.h \
h5b_model_private.h \
@@ -64,16 +65,16 @@ nodist_include_HEADERS =
#OBJEXT = o
# What to build... Will be determined by configure script.
lib_LIBRARIES = libH5Core.a
lib_LIBRARIES = libH5hut.a
# Listing of all possible targets that I may build.
EXTRA_LIBRARIES = libH5Core.a
EXTRA_LIBRARIES = libH5hut.a
# Header files that I wish to install in $(prefix)/include
include_HEADERS =
# Listing of sources
libH5Core_a_SOURCES = \
libH5hut_a_SOURCES = \
h5_attribs.c \
h5_errorhandling.c \
h5_fcmp.c \
@@ -86,6 +87,7 @@ libH5Core_a_SOURCES = \
h5_qsort_r.c \
h5_readwrite.c \
h5_syscall.c \
h5_lustre.c \
h5u_readwrite.c \
h5b_readwrite.c \
h5u_model.c \
@@ -110,12 +112,12 @@ libH5Core_a_SOURCES = \
h5t_store_trim.c \
h5t_tags.c
libH5Core_a_DEPENDENCIES = $(EXTRA_HEADERS)
libH5hut_a_DEPENDENCIES = $(EXTRA_HEADERS)
all: ../lib/libH5Core.a
all: ../lib/libH5hut.a
../lib/libH5Core.a: libH5Core.a
../lib/libH5hut.a: libH5hut.a
-cp $^ $@
$(libH5Core_a_OBJECTS): $(libH5Core_a_DEPENDENCIES)
$(libH5hut_a_OBJECTS): $(libH5hut_a_DEPENDENCIES)
src/h5core/h5_attribs.c
+12 -3
View File
@@ -31,10 +31,12 @@ h5_read_attrib (
const hid_t attrib_type, /*!< HDF5 type of attribute */
void* const attrib_value /*!< OUT: attribute value */
) {
if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f );
hid_t attrib_id;
hid_t space_id;
hid_t type_id;
hid_t mytype;
hid_t id;
TRY( _get_hdf5_obj_id(f, mode, &id) );
@@ -56,7 +58,7 @@ h5_read_attrib (
TRY( space_id = h5priv_get_hdf5_attribute_dataspace (f, attrib_id) );
TRY( h5priv_read_hdf5_attribute (f, attrib_id, type_id, attrib_value) );
TRY( h5priv_close_hdf5_dataspace(f, space_id) );
TRY( h5priv_close_hdf5_type(f, mytype) );
TRY( h5priv_close_hdf5_type(f, type_id) );
TRY( h5priv_close_hdf5_attribute (f, attrib_id) );
return H5_SUCCESS;
@@ -78,6 +80,10 @@ h5_write_attrib (
const void* attrib_value, /*!< value of attribute */
const hsize_t attrib_nelem /*!< number of elements (dimension) */
) {
if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f );
CHECK_WRITABLE_MODE( f );
hid_t space_id;
hid_t attrib_id;
hid_t type_id;
@@ -131,6 +137,9 @@ h5_get_attrib_info (
h5_int64_t* attrib_type, /*!< OUT: H5 type of attribute */
h5_size_t* attrib_nelem /*!< OUT: number of elements */
) {
if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f );
hid_t attrib_id;
hid_t mytype;
hid_t space_id;
@@ -178,7 +187,7 @@ h5_get_num_attribs (
h5_file_t *const f, /*!< handle to open file */
const char mode /*!< FILE or STEP flag */
) {
CHECK_FILEHANDLE (f);
if (mode != H5_ATTRIB_FILE) CHECK_TIMEGROUP( f );
hid_t id;
TRY( _get_hdf5_obj_id(f, mode, &id) );
return h5priv_get_num_hdf5_attribute (f, id);
src/h5core/h5_core_private.h
+9 -4
View File
@@ -1,14 +1,16 @@
#ifndef __H5_CORE_PRIVATE_H
#define __H5_CORE_PRIVATE_H
#define H5_DATANAME_LEN 128
#define H5_STEPNAME_LEN 128
/* WARNING! Changing these values will alter the data model and introduce
* file incompatibilities with previous versions. */
#define H5_DATANAME_LEN 64
#define H5_STEPNAME_LEN 64
#define H5_STEPNAME "Step"
#define H5_STEPWIDTH 1
#define H5_BLOCKNAME "Block"
#define H5_BLOCKNAME_X "0"
#define H5_BLOCKNAME_Y "0"
#define H5_BLOCKNAME_Z "0"
#define H5_BLOCKNAME_Y "1"
#define H5_BLOCKNAME_Z "2"
#include "h5_types_private.h"
@@ -20,6 +22,9 @@
#include "h5_qsort_private.h"
#include "h5_readwrite_private.h"
#include "h5_syscall_private.h"
#ifdef H5_USE_LUSTRE
#include "h5_lustre_private.h"
#endif
#include "h5b_types_private.h"
#include "h5u_types_private.h"
src/h5core/h5_errorhandling.c
+1 -1
View File
@@ -6,7 +6,7 @@
#include "h5_core_private.h"
static h5_errorhandler_t h5priv_errhandler = h5_report_errorhandler;
static h5_int32_t h5priv_debug_level = 0;
static h5_int32_t h5priv_debug_level = 1;
/*!
\ingroup h5_core
src/h5core/h5_hdf5.c
+91 -28
View File
@@ -1,3 +1,4 @@
#include <stdlib.h>
#include <string.h>
#include <hdf5.h>
@@ -55,7 +56,6 @@ h5priv_open_group (
const char* const group_name
) {
hid_t group_id;
herr_t herr = H5Gget_objinfo (loc_id, group_name, 1, NULL);
/*
check access modes:
@@ -66,7 +66,9 @@ h5priv_open_group (
H5_O_APPEND x x (append datasets to an existing group)
*/
if (herr >= 0) {
h5_err_t exists;
TRY( exists = h5priv_hdf5_link_exists(f, loc_id, group_name) );
if (exists > 0) {
h5_info (
f,
"Opening group %s/%s.",
@@ -87,7 +89,7 @@ h5priv_open_group (
return h5_error (
f,
H5_ERR_HDF5,
"Cannot open group \"%s/%s\".",
"Cannot open or create group %s/%s.",
h5_get_objname (loc_id),
group_name);
@@ -574,7 +576,7 @@ h5priv_close_hdf5_dataspace (
h5_file_t* const f,
const hid_t dataspace_id
) {
if (dataspace_id == 0 || dataspace_id == -1 || dataspace_id == H5S_ALL)
if (dataspace_id <= 0 || dataspace_id == H5S_ALL)
return H5_SUCCESS;
herr_t herr = H5Sclose (dataspace_id);
@@ -582,8 +584,7 @@ h5priv_close_hdf5_dataspace (
return h5_error(
f,
H5_ERR_HDF5,
"Cannot terminate access to dataspace \"%s\".",
h5_get_objname (dataspace_id));
"Cannot terminate access to dataspace!");
return H5_SUCCESS;
}
@@ -1202,12 +1203,71 @@ h5priv_delete_hdf5_link (
typedef struct op_data {
int queried_idx;
int cnt;
H5L_type_t type;
H5O_type_t type;
char *name;
size_t len;
char *prefix;
h5_file_t *f;
} op_data_t;
static H5O_type_t
_iter_op_get_obj_type (
h5_file_t *const f,
const hid_t g_id,
const char* name,
const H5L_info_t* info
) {
herr_t herr;
H5O_info_t objinfo;
if ( info->type == H5L_TYPE_EXTERNAL ) {
char *buf;
TRY( buf = h5priv_alloc(f, NULL, info->u.val_size) );
herr = H5Lget_val(g_id, name, buf,
info->u.val_size, H5P_DEFAULT);
if ( herr < 0 )
return h5_error(f,
H5_ERR_HDF5,
"Can't get external link for object '%s'!",
name);
const char *filename;
const char *objname;
herr = H5Lunpack_elink_val(buf, info->u.val_size, 0,
&filename, &objname);
if ( herr < 0 )
return h5_error(f,
H5_ERR_HDF5,
"Can't unpack external link for object '%s'!",
name);
h5_debug(f,
"Followed external link to file '%s' / object '%s'.",
filename, objname);
free(buf);
hid_t obj_id = H5Oopen(g_id, name, H5P_DEFAULT);
if ( obj_id < 0 )
return h5_error(f,
H5_ERR_HDF5,
"Can't open external link for object '%s'!",
name);
herr = H5Oget_info(obj_id, &objinfo);
}
else { // H5L_TYPE_HARD
herr = H5Oget_info_by_name(g_id, name, &objinfo, H5P_DEFAULT);
}
if ( herr < 0 )
return h5_error(f,
H5_ERR_HDF5,
"Can't query object with name '%s'!", name);
return objinfo.type;
}
static herr_t
iter_op_count (
hid_t g_id,
@@ -1216,10 +1276,9 @@ iter_op_count (
void* _op_data
) {
op_data_t* op_data = (op_data_t*)_op_data;
if (info->type != op_data->type) {
return 0;
}
H5O_type_t type;
TRY( type = _iter_op_get_obj_type(op_data->f, g_id, name, info) );
if ( type != op_data->type ) return 0;
op_data->cnt++;
return 0;
}
@@ -1232,9 +1291,9 @@ iter_op_idx (
void* _op_data
) {
op_data_t* op_data = (op_data_t*)_op_data;
if (info->type != op_data->type) {
return 0;
}
H5O_type_t type;
TRY( type = _iter_op_get_obj_type(op_data->f, g_id, name, info) );
if ( type != op_data->type ) return 0;
op_data->cnt++;
/* stop iterating if index is equal cnt */
if (op_data->queried_idx == op_data->cnt) {
@@ -1253,14 +1312,13 @@ iter_op_count_match (
void* _op_data
) {
op_data_t* op_data = (op_data_t*)_op_data;
if (info->type != op_data->type) {
return 0;
}
H5O_type_t type;
TRY( type = _iter_op_get_obj_type(op_data->f, g_id, name, info) );
if ( type != op_data->type ) return 0;
/* count if prefix matches */
if (strncmp (name, op_data->prefix, strlen(op_data->prefix)) == 0) {
op_data->cnt++;
}
return 0;
}
@@ -1271,7 +1329,7 @@ h5_get_num_hdf5_groups (
) {
op_data_t op_data;
memset (&op_data, 0, sizeof (op_data));
op_data.type = (H5L_type_t)H5G_GROUP;
op_data.type = H5O_TYPE_GROUP;
hsize_t start_idx = 0;
herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC,
&start_idx,
@@ -1294,7 +1352,8 @@ h5_get_num_hdf5_groups_matching_prefix (
) {
op_data_t op_data;
memset (&op_data, 0, sizeof (op_data));
op_data.type = (H5L_type_t)H5G_GROUP;
op_data.f = f;
op_data.type = H5O_TYPE_GROUP;
op_data.prefix = prefix;
hsize_t start_idx = 0;
herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC,
@@ -1316,13 +1375,16 @@ h5_get_hdf5_groupname_by_idx (
hid_t loc_id,
hsize_t idx,
char *name,
size_t size
size_t len
) {
op_data_t op_data;
memset (&op_data, 0, sizeof (op_data));
op_data.type = (H5L_type_t)H5G_GROUP;
op_data.f = f;
op_data.type = H5O_TYPE_GROUP;
op_data.cnt = -1;
op_data.queried_idx = idx;
op_data.name = name;
op_data.len = len;
hsize_t start_idx = 0;
herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC,
&start_idx,
@@ -1334,7 +1396,6 @@ h5_get_hdf5_groupname_by_idx (
"Cannot get name of group with index \"%lu\" in \"%s\".",
(long unsigned int)idx, h5_get_objname (loc_id));
}
strncpy (name, op_data.name, size);
return H5_SUCCESS;
}
@@ -1345,7 +1406,8 @@ h5_get_num_hdf5_datasets (
) {
op_data_t op_data;
memset (&op_data, 0, sizeof (op_data));
op_data.type = (H5L_type_t)H5G_DATASET;
op_data.f = f;
op_data.type = H5O_TYPE_DATASET;
hsize_t start_idx = 0;
herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC,
&start_idx,
@@ -1369,14 +1431,16 @@ h5_get_hdf5_datasetname_by_idx (
hid_t loc_id,
hsize_t idx,
char *name,
size_t size
size_t len
) {
return H5_SUCCESS;
op_data_t op_data;
memset (&op_data, 0, sizeof (op_data));
op_data.type = (H5L_type_t)H5G_DATASET;
op_data.f = f;
op_data.type = H5O_TYPE_DATASET;
op_data.cnt = -1;
op_data.queried_idx = idx;
op_data.name = name;
op_data.len = len;
hsize_t start_idx = 0;
herr_t herr = H5Literate (loc_id, H5_INDEX_NAME, H5_ITER_INC,
&start_idx,
@@ -1388,7 +1452,6 @@ h5_get_hdf5_datasetname_by_idx (
"Cannot get name of dataset with index \"%lu\" in \"%s\".",
(long unsigned int)idx, h5_get_objname (loc_id));
}
strncpy (name, op_data.name, size);
return H5_SUCCESS;
}
src/h5core/h5_lustre.c
+155
View File
@@ -0,0 +1,155 @@
#ifdef H5_USE_LUSTRE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/ioctl.h>
#define __USE_GNU
#include <fcntl.h>
#undef __USE_GNU
#include <lustre/liblustreapi.h>
#include "h5core/h5_core.h"
#include "h5_core_private.h"
#define MSG_HEADER "optimize for lustre: "
static void
_print_stripe_info(struct lov_user_md *lum)
{
fprintf (stderr, "lmm_magic: %u\n", (unsigned)lum->lmm_magic);
fprintf (stderr, "lmm_pattern: %u\n", (unsigned)lum->lmm_pattern);
fprintf (stderr, "lmm_object_id: %lu\n", (unsigned long)lum->lmm_object_id);
fprintf (stderr, "lmm_object_gr: %lu\n", (unsigned long)lum->lmm_object_gr);
fprintf (stderr, "lmm_stripe_size: %u\n", (unsigned)lum->lmm_stripe_size);
fprintf (stderr, "lmm_stripe_count: %u\n", (unsigned)lum->lmm_stripe_count);
fprintf (stderr, "lmm_stripe_offset: %u\n", (unsigned)lum->lmm_stripe_offset);
}
static ssize_t
_get_lustre_stripe_size ( const char *path )
{
size_t nbytes = sizeof(struct lov_user_md) +
INIT_ALLOC_NUM_OSTS * sizeof(struct lov_user_ost_data);
struct lov_user_md *lum;
TRY( lum = h5priv_alloc(f, NULL, nbytes) );
lum->lmm_magic = LOV_USER_MAGIC;
int fd = open64(path, O_RDONLY);
if (fd < 0) {
extern int errno;
if (errno == EINVAL)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: a flag is invalid!");
else if (errno == EACCES)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: access denied or file does not exist!");
else if (errno == ENAMETOOLONG)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: path is too long!");
else
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"open64: unspecific error!");
return -1;
}
int ret = ioctl(fd, LL_IOC_LOV_GETSTRIPE, lum);
if (ret == -1) {
extern int errno;
if (errno == EBADF)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"ioctl: bad file handle!");
else if (errno == EINVAL)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"ioctl: invalid argument!");
else if (errno == EIO)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"ioctl: physical I/O problem!");
else if (errno == ENOTTY)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"ioctl: file handle does not accept control functions!");
else if (errno == ENODEV)
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"ioctl: driver doesn't support control functions!");
else
h5_error(f, H5_ERR_INTERNAL, MSG_HEADER
"ioctl: unspecific error!");
return -1;
}
close(fd);
if ( f->myproc == 0 && h5_get_debuglevel() >= 5 )
_print_stripe_info(lum);
ssize_t stripe_size = (ssize_t)lum->lmm_stripe_size;
free(lum);
return stripe_size;
}
herr_t
h5_optimize_for_lustre (
h5_file_t *const f,
const char *filename
) {
ssize_t stripe_size;
if ( f->myproc == 0 )
{
char *path = malloc(strlen(filename)+4);
strcpy(path, filename);
/* check for existing file */
FILE *test = fopen(path, "r");
if (!test) {
/* use directory as path */
int i = strlen(path)-1;
while (i >= 0) {
if (path[i] != '/') path[i] = '\0';
else break;
i--;
}
if (strlen(path) == 0) sprintf(path, ".");
}
else fclose(test);
stripe_size = _get_lustre_stripe_size(path);
free(path);
}
TRY( h5priv_mpi_bcast(f, &stripe_size, 1, MPI_LONG_LONG, 0, f->comm) );
h5_info(f, "Found lustre stripe size of %lld bytes", (long long)stripe_size);
hsize_t btree_ik = (stripe_size - 4096) / 96;
hsize_t btree_bytes = 64 + 96*btree_ik;
h5_info(f,
"Setting HDF5 btree ik to %lld (= %lld bytes at rank 3)",
(long long)btree_ik, (long long)btree_bytes);
TRY( h5priv_set_hdf5_btree_ik_property(f, f->create_prop, btree_ik) );
/* set alignment to lustre stripe size */
TRY( h5priv_set_hdf5_alignment_property(f,
f->access_prop, 0, stripe_size) );
if (_verbose) {
fprintf(MSG_STREAM, MSG_HEADER "disabling metadata cache flushes.\n");
}
/* disable metadata cache flushes */
/* defer metadata writes */
H5AC_cache_config_t config;
config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
TRY( H5Pget_mdc_config( fapl_id, &config ) );
config.set_initial_size = 1;
config.initial_size = 16 * 1024 * 1024;
config.evictions_enabled = 0;
config.incr_mode = H5C_incr__off;
config.decr_mode = H5C_decr__off;
config.flash_incr_mode = H5C_flash_incr__off;
TRY( H5Pset_mdc_config( fapl_id, &config ) );
return H5_SUCCESS;
}
#endif // H5_USE_LUSTRE
src/h5core/h5_lustre_private.h
+10
View File
@@ -0,0 +1,10 @@
#ifndef __H5_LUSTRE_PRIVATE_H
#define __H5_LUSTRE_PRIVATE_H
h5_err_t
h5_optimize_for_lustre (
h5_file_t *const f,
const char *filename
);
#endif
src/h5core/h5_mpi.c
+23
View File
@@ -50,6 +50,29 @@ h5priv_mpi_send(
return H5_SUCCESS;
}
h5_err_t
h5priv_mpi_bcast (
h5_file_t* const f,
void* buf,
const int count,
const MPI_Datatype type,
const int root,
const MPI_Comm comm
) {
int err = MPI_Bcast(
buf,
count,
type,
root,
comm
);
if (err != MPI_SUCCESS)
return h5_error (f, H5_ERR_MPI, "Cannot perform broadcast");
return H5_SUCCESS;
}
h5_err_t
h5priv_mpi_sum (
h5_file_t* const f,
src/h5core/h5_mpi_private.h
+10
View File
@@ -24,6 +24,16 @@ h5priv_mpi_send(
const MPI_Comm comm
);
h5_err_t
h5priv_mpi_bcast (
h5_file_t* const f,
void* buf,
const int count,
const MPI_Datatype type,
const int root,
const MPI_Comm comm
);
h5_err_t
h5priv_mpi_sum (
h5_file_t* const f,
src/h5core/h5_openclose.c
+16 -70
View File
@@ -3,9 +3,6 @@
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#if H5_LUSTRE
#include <lustre/liblustreapi.h>
#endif
#include "h5core/h5_core.h"
#include "h5_core_private.h"
@@ -27,7 +24,7 @@ h5_check_filehandle (
h5_file_t* const f /*!< filehandle to check validity of */
) {
if (f == NULL || f->file == 0 || f->u == NULL || f->b == NULL || f->t == NULL) {
if (f == NULL || f->file < 0 || f->u == NULL || f->b == NULL || f->t == NULL) {
return h5_error (
f,
H5_ERR_BADFD,
@@ -59,13 +56,14 @@ h5priv_error_handler (
\return H5_SUCCESS or error code
*/
static h5_int64_t
static h5_err_t
h5upriv_open_file (
h5_file_t* const f /*!< IN: file handle */
) {
TRY( f->u = (h5u_fdata_t*)h5priv_alloc (f, NULL, sizeof (*f->u)) );
h5u_fdata_t *u = f->u;
u->shape = -1;
u->diskshape = H5S_ALL;
u->memshape = H5S_ALL;
u->viewstart = -1;
@@ -86,7 +84,7 @@ h5upriv_open_file (
\return H5_SUCCESS or error code
*/
static h5_int64_t
static h5_err_t
h5bpriv_open_file (
h5_file_t * const f /*!< IN: file handle */
) {
@@ -99,15 +97,13 @@ h5bpriv_open_file (
b = f->b;
memset (b, 0, sizeof (*b));
size_t size = f->nprocs * sizeof (b->user_layout[0]);
TRY( b->user_layout = h5priv_alloc (f, NULL, size) );
size = f->nprocs * sizeof (b->write_layout[0]);
TRY( b->write_layout = h5priv_alloc (f, NULL, size) );
size_t n = sizeof (struct h5b_partition) / sizeof (h5_int64_t);
TRY( h5priv_mpi_type_contiguous(f,
n, MPI_LONG_LONG, &b->partition_mpi_t) );
memset (b->user_layout, 0, sizeof(*b->user_layout));
memset (b->write_layout, 0, sizeof(*b->write_layout));
b->shape = -1;
b->diskshape = -1;
b->memshape = -1;
@@ -177,33 +173,10 @@ h5priv_open_file (
f->xfer_prop, H5FD_MPIO_COLLECTIVE) );
}
}
/* defer metadata writes */
H5AC_cache_config_t config;
config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
TRY( h5priv_get_hdf5_mdc_property(f, f->access_prop, &config) );
config.set_initial_size = 1;
config.initial_size = 16 * 1024 * 1024;
config.evictions_enabled = 0;
config.incr_mode = H5C_incr__off;
config.decr_mode = H5C_decr__off;
config.flash_incr_mode = H5C_flash_incr__off;
TRY( h5priv_set_hdf5_mdc_property(f, f->access_prop, &config) );
#endif /* PARALLEL_IO */
#if H5_LUSTRE
// set alignment
lov_user_md lum;
llapi_file_get_stripe(filename, &lum);
hsize_t stripe_size = (hsize_t)lum.lmm_stripe_size;
h5info(f, "Found lustre stripe size of %lld bytes", (long long)stripe_size);
TRY( h5priv_set_hdf5_alignment_property(f,
f->access_prop, 0, stripe_size) );
hsize_t btree_ik = (stripe_size - 4096) / 96;
hsize_t btree_bytes = 64 + 96*btree_ik;
h5info(f, "Using %lld bytes for HDF5 btree", (long long)btree_bytes);
TRY( h5priv_set_hdf5_btree_ik_property(f, f->create_prop, btree_ik) );
#ifdef H5_USE_LUSTRE
TRY( h5_optimize_for_lustre(f, filename) );
#endif
if (flags & H5_O_RDONLY) {
@@ -240,7 +213,7 @@ h5priv_open_file (
H5_ERR_HDF5,
"Cannot open file \"%s\" with mode \"%d\"",
filename, flags);
TRY( f->root_gid = h5priv_open_group (f, f->file, "/" ));
TRY( f->root_gid = h5priv_open_hdf5_group (f, f->file, "/" ) );
f->mode = flags;
f->step_gid = -1;
f->throttle = 0;
@@ -300,25 +273,16 @@ h5_open_file (
\return H5_SUCCESS or error code
*/
static h5_int64_t
static h5_err_t
h5upriv_close_file (
h5_file_t* const f /*!< file handle */
) {
struct h5u_fdata* u = f->u;
f->__errno = H5_SUCCESS;
if(u->shape != H5S_ALL) {
TRY( h5priv_close_hdf5_dataspace (f, u->shape) );
u->shape = 0;
}
if(u->diskshape != H5S_ALL) {
TRY( h5priv_close_hdf5_dataspace (f, u->diskshape) );
u->diskshape = 0;
}
if(u->memshape != H5S_ALL) {
TRY( h5priv_close_hdf5_dataspace (f, u->memshape) );
u->memshape = 0;
}
TRY( h5priv_close_hdf5_dataspace (f, u->shape) );
TRY( h5priv_close_hdf5_dataspace (f, u->diskshape) );
TRY( h5priv_close_hdf5_dataspace (f, u->memshape) );
TRY( h5priv_close_hdf5_property (f, u->dcreate_prop) );
free (f->u);
f->u = NULL;
@@ -336,13 +300,14 @@ h5upriv_close_file (
\return H5_SUCCESS or error code
*/
static h5_int64_t
static h5_err_t
h5bpriv_close_file (
h5_file_t* const f /*!< IN: file handle */
) {
struct h5b_fdata* b = f->b;
TRY( h5priv_close_hdf5_group (f, b->block_gid) );
TRY( h5priv_close_hdf5_group (f, b->field_gid) );
TRY( h5priv_close_hdf5_dataspace (f, b->shape) );
TRY( h5priv_close_hdf5_dataspace (f, b->diskshape) );
TRY( h5priv_close_hdf5_dataspace (f, b->memshape) );
@@ -490,25 +455,6 @@ h5_get_num_steps(
f->prefix_step_name);
}
/*!
\ingroup h5_core_filehandling
Check whether step with number \c stepno exists.
\return True (value != 0) if step with \c stepno exists.
\return False (0) otherwise
*/
h5_err_t
h5_has_step (
h5_file_t* const f, /*!< file handle */
h5_id_t stepno /*!< step number to check */
) {
char name[128];
sprintf (name, "%s#%0*ld",
f->prefix_step_name, f->width_step_idx, (long)stepno);
return (H5Gget_info_by_name (f->file, name, NULL, H5P_DEFAULT) >= 0);
}
/*!
\ingroup h5_core_filehandling
src/h5core/h5_readwrite.c
+11 -10
View File
@@ -200,7 +200,7 @@ h5priv_close_step (
static h5_err_t
_set_step (
h5_file_t* const f,
const h5_int64_t step_idx /*!< [in] Step to set. */
const h5_id_t step_idx /*!< [in] Step to set. */
) {
f->step_idx = step_idx;
@@ -219,10 +219,10 @@ _set_step (
return H5_SUCCESS;
}
h5_int64_t
h5_err_t
h5_set_step (
h5_file_t* const f, /*!< [in] Handle to open file */
const h5_int64_t step_idx /*!< [in] Step to set. */
h5_file_t* const f, /*!< [in] Handle to open file */
const h5_id_t step_idx /*!< [in] Step to set. */
) {
TRY( h5priv_close_step (f) );
@@ -290,15 +290,15 @@ h5_get_dataset_type(
}
h5_err_t
h5_has_index (
h5_has_step (
h5_file_t* const f, /*!< [in] Handle to open file */
const h5_int64_t step /*!< [in] Step number to query */
const h5_id_t step /*!< [in] Step number to query */
) {
char name[2*H5_STEPNAME_LEN];
sprintf (name,
"%s#%0*lld",
f->prefix_step_name, f->width_step_idx, (long long)step);
return (H5Gget_objinfo(f->file, name, 1, NULL) >= 0);
return h5priv_hdf5_link_exists(f, f->file, name);
}
h5_err_t
@@ -309,8 +309,9 @@ h5_normalize_dataset_name (
) {
if ( strlen(name) > H5_DATANAME_LEN ) {
strncpy ( name2, name, H5_DATANAME_LEN - 1 );
strncpy ( name2, name, H5_DATANAME_LEN-1 );
name2[H5_DATANAME_LEN-1] = '\0';
h5_warn (f, "Truncated name '%s' to '%s'.", name, name2);
} else {
strcpy ( name2, name );
}
@@ -318,8 +319,8 @@ h5_normalize_dataset_name (
if ( strcmp( name2, H5_BLOCKNAME ) == 0 ) {
h5_error (f,
H5_ERR_INVAL,
"Can't create dataset or field with name '%s' because it is "
"reserved by H5Block.",
"Can't create dataset or field with name '%s' because "
"it is reserved by H5Block.",
H5_BLOCKNAME);
}
src/h5core/h5_types_private.h
+1 -1
View File
@@ -29,7 +29,7 @@ struct h5_file {
hid_t xfer_prop; /* dataset transfer properties */
hid_t access_prop; /* file access properties */
hid_t create_prop; /* file create properties */
hid_t root_gid; /* id of root group */
hid_t root_gid; /* id of root */
hid_t step_gid; /* id of current step */
/* step internal data */
src/h5core/h5b_attribs.c
+1 -1
View File
@@ -11,7 +11,7 @@ h5_write_field_attrib (
const h5_int64_t attrib_nelem /*!< IN: number of elements */
) {
TRY( h5bpriv_open_field_group(f, field_name) );
TRY( h5bpriv_create_field_group(f, field_name) );
TRY( h5_write_attrib (
f,
src/h5core/h5b_errorhandling_private.h
+2 -2
View File
@@ -6,12 +6,12 @@
return h5_error( \
f, \
H5_ERR_LAYOUT, \
"No layout defined.")
"No view has been defined!")
#define HANDLE_H5_LAYOUT_ERR( f ) \
h5_error( \
f, \
H5_ERR_LAYOUT, \
"Bad layout.");
"Bad view!");
#endif
src/h5core/h5b_model.c
+176 -157
View File
@@ -17,75 +17,46 @@
static void
_normalize_partition (
struct h5b_partition *p /*!< IN/OUT: partition */
h5b_partition_t *const p /*!< IN/OUT: partition */
) {
h5_int64_t x;
h5_size_t tmp;
if ( p->i_start > p->i_end ) {
x = p->i_start;
tmp = p->i_start;
p->i_start = p->i_end;
p->i_end = x;
p->i_end = tmp;
}
if ( p->j_start > p->j_end ) {
x = p->j_start;
tmp = p->j_start;
p->j_start = p->j_end;
p->j_end = x;
p->j_end = tmp;
}
if ( p->k_start > p->k_end ) {
x = p->k_start;
tmp = p->k_start;
p->k_start = p->k_end;
p->k_end = x;
p->k_end = tmp;
}
}
/*!
\ingroup h5block_private
\internal
Gather layout to all processors
\return H5_SUCCESS or error code
*/
static h5_err_t
_allgather (
h5_file_t *const f /*!< IN: file handle */
) {
#ifdef PARALLEL_IO
struct h5b_partition *partition = &f->b->user_layout[f->myproc];
struct h5b_partition *layout = f->b->user_layout;
TRY( h5priv_mpi_allgather(f,
partition, 1, f->b->partition_mpi_t,
layout, 1, f->b->partition_mpi_t, f->comm) );
#endif
return H5_SUCCESS;
}
/*!
\ingroup h5block_private
\internal
Get dimension sizes of block. These informations are stored inside the
block structure.
*/
/* MLH: this could be improved with an MPI_Reduce and MAX operator...
* but the user_layout array-of-structs would need to be a struct-of-arrays */
static void
_get_dimension_sizes (
h5_file_t *const f /*!< IN: file handle */
_get_max_dimensions (
h5_file_t *const f,
h5b_partition_t *const user_layout
) {
int proc;
struct h5b_fdata *b = f->b;
struct h5b_partition *partition = b->user_layout;
h5b_fdata_t *b = f->b;
h5b_partition_t *p = user_layout;
b->i_max = 0;
b->j_max = 0;
b->k_max = 0;
for ( proc = 0; proc < f->nprocs; proc++, partition++ ) {
if ( partition->i_end > b->i_max ) b->i_max = partition->i_end;
if ( partition->j_end > b->j_max ) b->j_max = partition->j_end;
if ( partition->k_end > b->k_max ) b->k_max = partition->k_end;
for ( proc = 0; proc < f->nprocs; proc++, p++ ) {
if ( p->i_end > b->i_max ) b->i_max = p->i_end;
if ( p->j_end > b->j_max ) b->j_max = p->j_end;
if ( p->k_end > b->k_max ) b->k_max = p->k_end;
}
}
@@ -105,8 +76,8 @@ _get_dimension_sizes (
*/
static int
_have_ghostzone (
const struct h5b_partition *p, /*!< IN: partition \c p */
const struct h5b_partition *q /*!< IN: partition \c q */
const h5b_partition_t *const p, /*!< IN: partition \c p */
const h5b_partition_t *const q /*!< IN: partition \c q */
) {
return ( ! ( _NO_GHOSTZONE ( p, q ) || _NO_GHOSTZONE ( q, p ) ) );
}
@@ -122,7 +93,7 @@ _have_ghostzone (
*/
static h5_int64_t
_volume_of_partition (
const struct h5b_partition *p /*!< IN: partition */
const h5b_partition_t *const p /*!< IN: partition */
) {
return (p->i_end - p->i_start)
* (p->j_end - p->j_start)
@@ -144,8 +115,8 @@ _volume_of_partition (
*/
static h5_int64_t
_volume_of_ghostzone (
const struct h5b_partition *p, /*!< IN: ptr to first partition */
const struct h5b_partition *q /*!< IN: ptr to second partition */
const h5b_partition_t *const p, /*!< IN: ptr to first partition */
const h5b_partition_t *const q /*!< IN: ptr to second partition */
) {
h5_int64_t dx = MIN ( p->i_end, q->i_end )
@@ -171,8 +142,8 @@ _volume_of_ghostzone (
*/
static h5_int64_t
_dissolve_X_ghostzone (
struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */
struct h5b_partition *q /*!< IN/OUT: ptr to second partition */
h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */
h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */
) {
if ( p->i_start > q->i_start )
@@ -199,8 +170,8 @@ _dissolve_X_ghostzone (
*/
static h5_int64_t
_dissolve_Y_ghostzone (
struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */
struct h5b_partition *q /*!< IN/OUT: ptr to second partition */
h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */
h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */
) {
if ( p->j_start > q->j_start )
@@ -227,8 +198,8 @@ _dissolve_Y_ghostzone (
*/
static h5_int64_t
_dissolve_Z_ghostzone (
struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */
struct h5b_partition *q /*!< IN/OUT: ptr to second partition */
h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */
h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */
) {
if ( p->k_start > q->k_start )
@@ -256,17 +227,17 @@ _dissolve_Z_ghostzone (
\return H5_SUCCESS or error code.
*/
static h5_int64_t
static h5_err_t
_dissolve_ghostzone (
h5_file_t *const f,
struct h5b_partition *p, /*!< IN/OUT: ptr to first partition */
struct h5b_partition *q /*!< IN/OUT: ptr to second partition */
h5b_partition_t *const p, /*!< IN/OUT: ptr to first partition */
h5b_partition_t *const q /*!< IN/OUT: ptr to second partition */
) {
struct h5b_partition p_;
struct h5b_partition q_;
struct h5b_partition p_best;
struct h5b_partition q_best;
h5b_partition_t p_;
h5b_partition_t q_;
h5b_partition_t p_best;
h5b_partition_t q_best;
h5_int64_t vol;
h5_int64_t max_vol = 0;
@@ -338,36 +309,36 @@ _dissolve_ghostzone (
\return H5_SUCCESS or error code.
*/
static h5_int64_t
static h5_err_t
_dissolve_ghostzones (
h5_file_t *const f /*!< IN: file handle */
h5_file_t *const f,
const h5b_partition_t *const user_layout,
h5b_partition_t *const write_layout
) {
struct h5b_fdata *b = f->b;
struct h5b_partition *p;
struct h5b_partition *q;
h5b_partition_t *p;
h5b_partition_t *q;
int proc_p, proc_q;
struct list {
struct list *prev;
struct list *next;
struct h5b_partition *p;
struct h5b_partition *q;
h5b_partition_t *p;
h5b_partition_t *q;
h5_int64_t vol;
} *p_begin, *p_el, *p_max, *p_end, *p_save;
memcpy ( b->write_layout, b->user_layout,
f->nprocs * sizeof (*f->b->user_layout) );
memcpy( write_layout, user_layout, f->nprocs*sizeof(h5b_partition_t) );
TRY( p_begin = (struct list*)h5priv_alloc(f, NULL, sizeof(*p_begin)) );
p_max = p_end = p_begin;
memset ( p_begin, 0, sizeof ( *p_begin ) );
memset( p_begin, 0, sizeof ( *p_begin ) );
for ( proc_p = 0, p = b->write_layout;
for ( proc_p = 0, p = write_layout;
proc_p < f->nprocs-1;
proc_p++, p++ ) {
for ( proc_q = proc_p+1, q = &b->write_layout[proc_q];
for ( proc_q = proc_p+1, q = &write_layout[proc_q];
proc_q < f->nprocs;
proc_q++, q++ ) {
@@ -416,45 +387,6 @@ _dissolve_ghostzones (
}
free ( p_begin );
p = &b->user_layout[f->myproc];
h5_debug (f,
"PROC[%d]: User layout: %lld:%lld, %lld:%lld, %lld:%lld",
f->myproc,
(long long)p->i_start, (long long)p->i_end,
(long long)p->j_start, (long long)p->j_end,
(long long)p->k_start, (long long)p->k_end );
/* more detailed debug output: all procs report their view
of all other procs */
for ( proc_p = 0, p = b->user_layout;
proc_p < f->nprocs;
proc_p++, p++ ) {
h5_debug (f,
"PROC[%d]: proc[%d]: User layout: %lld:%lld, %lld:%lld, %lld:%lld ",
f->myproc, proc_p,
(long long)p->i_start, (long long)p->i_end,
(long long)p->j_start, (long long)p->j_end,
(long long)p->k_start, (long long)p->k_end );
}
p = &b->write_layout[f->myproc];
h5_debug (f,
"PROC[%d]: Ghost-zone layout: %lld:%lld, %lld:%lld, %lld:%lld",
f->myproc,
(long long)p->i_start, (long long)p->i_end,
(long long)p->j_start, (long long)p->j_end,
(long long)p->k_start, (long long)p->k_end );
/* more detailed debug output: all procs report their view
of all other procs */
for ( proc_p = 0, p = b->write_layout;
proc_p < f->nprocs;
proc_p++, p++ ) {
h5_debug (f,
"PROC[%d]: proc[%d]: Ghost-zone layout: %lld:%lld, %lld:%lld, %lld:%lld ",
f->myproc, proc_p,
(long long)p->i_start, (long long)p->i_end,
(long long)p->j_start, (long long)p->j_end,
(long long)p->k_start, (long long)p->k_end );
}
return H5_SUCCESS;
}
@@ -482,15 +414,31 @@ h5bpriv_open_block_group (
h5_file_t *const f /*!< IN: file handle */
) {
struct h5b_fdata *b = f->b;
h5b_fdata_t *b = f->b;
if ( b->block_gid >= 0) {
TRY( h5priv_close_hdf5_group(f, b->block_gid) );
b->block_gid = -1;
}
TRY( h5priv_close_hdf5_group(f, b->block_gid) );
b->block_gid = h5priv_open_hdf5_group(f, f->step_gid, H5_BLOCKNAME);
if (f->b->block_gid < 0)
return h5_error(f,
H5_ERR_INVAL,
"Time step does not contain H5Block data!");
if ( b->block_gid < 0 ) {
TRY( b->block_gid = h5priv_open_hdf5_group(f,
return H5_SUCCESS;
}
static h5_err_t
_create_block_group (
h5_file_t *const f /*!< IN: file handle */
) {
h5_err_t exists;
TRY( exists = h5priv_hdf5_link_exists(f, f->step_gid, H5_BLOCKNAME) );
if (exists > 0) {
TRY( h5bpriv_open_block_group(f) );
} else {
TRY( h5priv_close_hdf5_group(f, f->b->block_gid) );
TRY( f->b->block_gid = h5priv_create_hdf5_group(f,
f->step_gid, H5_BLOCKNAME) );
}
@@ -506,10 +454,10 @@ h5bpriv_have_field_group (
char name2[H5_DATANAME_LEN];
h5_normalize_dataset_name(f, name, name2);
TRY( h5bpriv_open_block_group ( f ) );
TRY( h5bpriv_open_block_group(f) );
h5_err_t exists;
TRY( exists = h5priv_hdf5_have_link(f, f->b->block_gid, name2) );
TRY( exists = h5priv_hdf5_link_exists(f, f->b->block_gid, name2) );
return exists;
}
@@ -523,16 +471,44 @@ h5bpriv_open_field_group (
char name2[H5_DATANAME_LEN];
h5_normalize_dataset_name(f, name, name2);
if ( f->b->field_gid >= 0 ) {
TRY( h5priv_close_hdf5_group(f, f->b->field_gid) );
}
TRY( h5bpriv_open_block_group ( f ) );
TRY( f->b->field_gid = h5priv_open_hdf5_group(f, f->b->block_gid, name2) );
TRY( h5priv_close_hdf5_group(f, f->b->field_gid) );
TRY( h5bpriv_open_block_group(f) );
f->b->field_gid = h5priv_open_hdf5_group(f, f->b->block_gid, name2);
if (f->b->field_gid < 0)
return h5_error(f,
H5_ERR_INVAL,
"Field '%s' does not exist!", name2);
return H5_SUCCESS;
}
h5_err_t
h5bpriv_create_field_group (
h5_file_t *const f, /*!< IN: file handle */
const char *name /*!< IN: name of field group to create */
) {
h5b_fdata_t *b = f->b;
TRY( _create_block_group(f) );
char name2[H5_DATANAME_LEN];
h5_normalize_dataset_name(f, name, name2);
h5_err_t exists;
TRY( exists = h5priv_hdf5_link_exists(f, b->block_gid, name2) );
if (exists > 0) {
TRY( h5bpriv_open_field_group(f, name2) );
} else {
TRY( h5priv_close_hdf5_group(f, f->b->field_gid) );
TRY( b->field_gid = h5priv_create_hdf5_group(f,
b->block_gid, name2) );
}
return H5_SUCCESS;
}
h5_err_t
h5b_3d_set_view (
h5_file_t *const f, /*!< IN: File handle */
@@ -544,22 +520,59 @@ h5b_3d_set_view (
const h5_size_t k_end /*!< IN: end index of \c k */
) {
struct h5b_fdata *b = f->b;
struct h5b_partition *p = &b->user_layout[f->myproc];
h5b_partition_t *p = f->b->user_layout;
p->i_start = i_start;
p->i_end = i_end;
p->j_start = j_start;
p->j_end = j_end;
p->k_start = k_start;
p->k_end = k_end;
_normalize_partition(p);
_normalize_partition ( p );
TRY( _allgather( f ) );
_get_dimension_sizes ( f );
TRY( _dissolve_ghostzones ( f ) );
TRY( _release_hyperslab ( f ) );
#ifdef PARALLEL_IO
h5b_fdata_t *b = f->b;
h5b_partition_t *user_layout;
h5b_partition_t *write_layout;
size_t size = f->nprocs * sizeof (h5b_partition_t);
TRY( user_layout = h5priv_alloc (f, NULL, size) );
TRY( write_layout = h5priv_alloc (f, NULL, size) );
TRY( h5priv_mpi_allgather(f,
p, 1, f->b->partition_mpi_t,
user_layout, 1, f->b->partition_mpi_t, f->comm) );
_get_max_dimensions(f, user_layout);
TRY( _dissolve_ghostzones(f, user_layout, write_layout) );
b->user_layout[0] = user_layout[f->myproc];
b->write_layout[0] = write_layout[f->myproc];
b->have_layout = 1;
p = b->user_layout;
h5_debug (f,
"[%d] User layout: %lld:%lld, %lld:%lld, %lld:%lld",
f->myproc,
(long long)p->i_start, (long long)p->i_end,
(long long)p->j_start, (long long)p->j_end,
(long long)p->k_start, (long long)p->k_end );
p = b->write_layout;
h5_debug (f,
"[%d] Ghost-zone layout: %lld:%lld, %lld:%lld, %lld:%lld",
f->myproc,
(long long)p->i_start, (long long)p->i_end,
(long long)p->j_start, (long long)p->j_end,
(long long)p->k_start, (long long)p->k_end );
free(user_layout);
free(write_layout);
TRY( h5bpriv_release_hyperslab(f) );
#endif
return H5_SUCCESS;
}
@@ -597,7 +610,7 @@ h5b_3d_get_chunk (
CHECK_TIMEGROUP ( f );
struct h5b_fdata *b = f->b;
h5b_fdata_t *b = f->b;
TRY( h5bpriv_open_field_group ( f, field_name ) );
@@ -639,7 +652,7 @@ h5b_3d_get_view (
if ( ( proc < 0 ) || ( proc >= f->nprocs ) )
return h5_error(f, H5_ERR_INVAL, "Invalid processor id %d!", proc);
struct h5b_partition *p = &f->b->user_layout[(size_t)proc];
h5b_partition_t *p = &f->b->user_layout[(size_t)proc];
*i_start = p->i_start;
*i_end = p->i_end;
@@ -666,7 +679,7 @@ h5b_3d_get_reduced_view (
if ( ( proc < 0 ) || ( proc >= f->nprocs ) )
return h5_error(f, H5_ERR_INVAL, "Invalid processor id %d!", proc);
struct h5b_partition *p = &f->b->write_layout[(size_t)proc];
h5b_partition_t *p = &f->b->write_layout[(size_t)proc];
*i_start = p->i_start;
*i_end = p->i_end;
@@ -686,7 +699,7 @@ h5b_3d_get_proc (
const h5_int64_t k /*!< IN: \c k coordinate */
) {
struct h5b_partition *layout = f->b->write_layout;
h5b_partition_t *layout = f->b->write_layout;
int proc;
for ( proc = 0; proc < f->nprocs; proc++, layout++ ) {
@@ -704,6 +717,8 @@ h5b_get_num_fields (
h5_file_t *const f /*!< IN: File handle */
) {
CHECK_TIMEGROUP( f );
TRY( h5bpriv_open_block_group(f) );
return h5priv_get_num_objs_in_hdf5_group( f, f->b->block_gid );
}
@@ -712,14 +727,16 @@ h5_err_t
h5b_get_field_info_by_name (
h5_file_t *const f, /*!< IN: file handle */
const char *name, /*!< OUT: field name */
h5_size_t *grid_rank, /*!< OUT: grid rank */
h5_size_t *grid_dims, /*!< OUT: grid dimensions */
h5_size_t *field_rank, /*!< OUT: field rank */
h5_size_t *field_dims, /*!< OUT: field dimensions */
h5_size_t *elem_rank, /*!< OUT: element rank */
h5_int64_t *type /*!< OUT: datatype */
) {
CHECK_TIMEGROUP( f );
hsize_t dims[16]; /* give it plenty of space even though we don't expect rank > 3 */
hsize_t _grid_rank, _field_rank;
hsize_t _field_rank, _elem_rank;
h5_size_t i, j;
TRY( h5bpriv_open_field_group(f, name) );
@@ -731,18 +748,18 @@ h5b_get_field_info_by_name (
f->b->field_gid, H5_BLOCKNAME_X) );
TRY( dataspace_id = h5priv_get_hdf5_dataset_space(f, dataset_id) );
TRY( _grid_rank = h5priv_get_dims_of_hdf5_dataspace(f,
TRY( _field_rank = h5priv_get_dims_of_hdf5_dataspace(f,
dataspace_id, dims, NULL) );
if ( grid_rank ) *grid_rank = (h5_size_t) _grid_rank;
if ( field_rank ) *field_rank = (h5_size_t) _field_rank;
if ( grid_dims ) {
for ( i = 0, j = _grid_rank-1; i < _grid_rank; i++, j-- )
grid_dims[i] = (h5_size_t)dims[j];
if ( field_dims ) {
for ( i = 0, j = _field_rank-1; i < _field_rank; i++, j-- )
field_dims[i] = (h5_size_t)dims[j];
}
TRY( _field_rank = h5priv_get_num_objs_in_hdf5_group(f,
f->b->block_gid) );
if ( field_rank ) *field_rank = (h5_size_t) _field_rank;
TRY( _elem_rank = h5priv_get_num_objs_in_hdf5_group(f,
f->b->field_gid) );
if ( elem_rank ) *elem_rank = (h5_size_t) _elem_rank;
hid_t h5type;
TRY( h5type = h5priv_get_hdf5_dataset_type(f, dataset_id) );
@@ -762,12 +779,14 @@ h5b_get_field_info (
const h5_size_t idx, /*!< IN: index of field */
char *name, /*!< OUT: field name */
const h5_size_t len_name, /*!< IN: buffer size */
h5_size_t *grid_rank, /*!< OUT: grid rank */
h5_size_t *grid_dims, /*!< OUT: grid dimensions */
h5_size_t *field_rank, /*!< OUT: field rank */
h5_size_t *field_dims, /*!< OUT: field dimensions */
h5_size_t *elem_rank, /*!< OUT: element rank */
h5_int64_t *type /*!< OUT: datatype */
) {
CHECK_TIMEGROUP( f );
TRY( h5bpriv_open_block_group(f) );
TRY( h5priv_get_hdf5_objname_by_idx(
f,
@@ -777,6 +796,6 @@ h5b_get_field_info (
(size_t)len_name) );
return h5b_get_field_info_by_name(f,
name, grid_rank, grid_dims, field_rank, type);
name, field_rank, field_dims, elem_rank, type);
}
src/h5core/h5b_model_private.h
+6
View File
@@ -13,6 +13,12 @@ h5bpriv_open_field_group (
const char *name
);
h5_err_t
h5bpriv_create_field_group (
h5_file_t *const f, /*!< IN: file handle */
const char *name /*!< IN: name of field group to create */
);
h5_err_t
h5bpriv_open_block_group (
h5_file_t *const f /*!< IN: file handle */
src/h5core/h5b_readwrite.c
+19 -53
View File
@@ -1,50 +1,6 @@
#include "h5core/h5_core.h"
#include "h5_core_private.h"
static h5_err_t
_create_block_group (
h5_file_t *const f /*!< IN: file handle */
) {
h5_err_t exists;
TRY( exists = h5priv_hdf5_link_exists(f, f->step_gid, H5_BLOCKNAME) );
if (exists > 0) {
TRY( h5bpriv_open_block_group(f) );
} else {
TRY( f->b->block_gid = h5priv_create_hdf5_group(f,
f->step_gid, H5_BLOCKNAME) );
}
return H5_SUCCESS;
}
static h5_err_t
_create_field_group (
h5_file_t *const f, /*!< IN: file handle */
const char *name /*!< IN: name of field group to create */
) {
struct h5b_fdata *b = f->b;
TRY( _create_block_group(f) );
char name2[H5_DATANAME_LEN];
h5_normalize_dataset_name(f, name, name2);
h5_err_t exists;
TRY( exists = h5priv_hdf5_link_exists(f, b->block_gid, name2) );
if (exists > 0) {
TRY( h5bpriv_open_field_group(f, name2) );
} else {
TRY( b->field_gid = h5priv_create_hdf5_group(f,
b->block_gid, name2) );
}
return H5_SUCCESS;
}
static h5_err_t
_select_hyperslab_for_writing (
h5_file_t *const f /*!< IN: file handle */
@@ -55,9 +11,9 @@ _select_hyperslab_for_writing (
*/
if ( f->b->shape >= 0 ) return H5_SUCCESS;
struct h5b_fdata *b = f->b;
struct h5b_partition *p = &b->write_layout[f->myproc];
struct h5b_partition *q = &b->user_layout[f->myproc];
h5b_fdata_t *b = f->b;
h5b_partition_t *p = b->write_layout;
h5b_partition_t *q = b->user_layout;
int rank = 3;
@@ -156,7 +112,7 @@ _write_data (
) {
hid_t dataset;
struct h5b_fdata *b = f->b;
h5b_fdata_t *b = f->b;
h5_err_t exists;
TRY( exists = h5priv_hdf5_link_exists (f, b->field_gid, data_name) );
@@ -203,7 +159,10 @@ h5b_write_scalar_data (
const void *data, /*!< IN: data to write */
const hid_t type /*!< IN: data type */
) {
TRY( _create_field_group(f, field_name) );
CHECK_TIMEGROUP( f );
CHECK_WRITABLE_MODE( f );
CHECK_LAYOUT( f );
TRY( h5bpriv_create_field_group(f, field_name) );
TRY( _select_hyperslab_for_writing(f) );
TRY( _write_data(f, field_name, H5_BLOCKNAME_X, data, type) );
return H5_SUCCESS;
@@ -218,7 +177,10 @@ h5b_write_vector3d_data (
const void *zdata, /*!< IN: z data to write */
const hid_t type /*!< IN: data type */
) {
TRY( _create_field_group(f, field_name) );
CHECK_TIMEGROUP( f );
CHECK_WRITABLE_MODE( f );
CHECK_LAYOUT( f );
TRY( h5bpriv_create_field_group(f, field_name) );
TRY( _select_hyperslab_for_writing(f) );
TRY( _write_data(f, field_name, H5_BLOCKNAME_X, xdata, type) );
TRY( _write_data(f, field_name, H5_BLOCKNAME_Y, ydata, type) );
@@ -232,8 +194,8 @@ _select_hyperslab_for_reading (
const hid_t dataset
) {
struct h5b_fdata *b = f->b;
struct h5b_partition *p = &b->user_layout[f->myproc];
h5b_fdata_t *b = f->b;
h5b_partition_t *p = b->user_layout;
int rank;
hsize_t field_dims[3];
hsize_t start[3] = {
@@ -314,7 +276,7 @@ _read_data (
) {
hid_t dataset;
struct h5b_fdata *b = f->b;
h5b_fdata_t *b = f->b;
TRY( dataset = h5priv_open_hdf5_dataset(f, b->field_gid, data_name) );
TRY( _select_hyperslab_for_reading(f, dataset) );
@@ -337,6 +299,8 @@ h5b_read_scalar_data (
void *data, /*!< OUT: read bufer */
const hid_t type /*!< IN: data type */
) {
CHECK_TIMEGROUP( f );
CHECK_LAYOUT( f );
TRY( h5bpriv_open_field_group(f, field_name) );
TRY( _read_data(f, field_name, H5_BLOCKNAME_X, data, type) );
return H5_SUCCESS;
@@ -351,6 +315,8 @@ h5b_read_vector3d_data (
void *zdata, /*!< IN: z data to write */
const hid_t type /*!< IN: data type */
) {
CHECK_TIMEGROUP( f );
CHECK_LAYOUT( f );
TRY( h5bpriv_open_field_group(f, field_name) );
TRY( _read_data(f, field_name, H5_BLOCKNAME_X, xdata, type) );
TRY( _read_data(f, field_name, H5_BLOCKNAME_Y, ydata, type) );
src/h5core/h5b_types_private.h
+3 -2
View File
@@ -15,8 +15,8 @@ struct h5b_fdata {
h5_size_t i_max;
h5_size_t j_max;
h5_size_t k_max;
struct h5b_partition* user_layout;
struct h5b_partition* write_layout;
struct h5b_partition user_layout[1];
struct h5b_partition write_layout[1];
int have_layout;
hid_t shape;
@@ -29,4 +29,5 @@ struct h5b_fdata {
MPI_Datatype partition_mpi_t;
};
typedef struct h5b_fdata h5b_fdata_t;
typedef struct h5b_partition h5b_partition_t;
#endif
src/h5core/h5t_openclose.c
+37 -20
View File
@@ -313,6 +313,18 @@ init_fdata (
t->dsinfo_num_elems_on_level.chunk_dims) );
t->dsinfo_num_elems_on_level.access_prop = H5P_DEFAULT;
/* initialize pointers */
t->elems.data = NULL;
t->num_elems = NULL;
t->elems_ldta = NULL;
t->num_elems_on_level = NULL;
t->map_elem_g2l.items = NULL;
t->vertices = NULL;
t->vertices_data = NULL;
t->num_vertices = NULL;
t->map_vertex_g2l.items = NULL;
t->mtags.names = NULL;
return H5_SUCCESS;
}
@@ -348,25 +360,6 @@ h5tpriv_open_file (
return H5_SUCCESS;
}
/*!
\ingroup h5_private
\internal
De-initialize topological internal structure. Open HDF5 objects are
closed and allocated memory freed.
\return H5_SUCCESS or error code
*/
h5_err_t
h5tpriv_close_file (
h5_file_t* const f /*!< IN: file handle */
) {
TRY( h5t_close_mesh (f) );
return H5_SUCCESS;
}
h5_err_t
h5tpriv_init_step (
h5_file_t* const f
@@ -533,7 +526,9 @@ h5_err_t
h5t_close_mesh (
h5_file_t* const f
) {
TRY( h5tpriv_write_mesh (f) );
if (!(f->mode & H5_O_RDONLY)) {
TRY( h5tpriv_write_mesh (f) );
}
TRY( release_memory (f) );
TRY( init_fdata (f) );
@@ -575,3 +570,25 @@ h5tpriv_alloc_num_vertices (
return H5_SUCCESS;
}
/*!
\ingroup h5_private
\internal
De-initialize topological internal structure. Open HDF5 objects are
closed and allocated memory freed.
\return H5_SUCCESS or error code
*/
h5_err_t
h5tpriv_close_file (
h5_file_t* const f /*!< IN: file handle */
) {
TRY( release_memory (f) );
TRY( h5t_close_mesh (f) );
return H5_SUCCESS;
}
src/h5core/h5u_model.c
+64 -102
View File
@@ -1,45 +1,11 @@
#include "h5core/h5_core.h"
#include "h5_core_private.h"
h5_int64_t
h5_ssize_t
h5u_get_num_particles (
h5_file_t *const f /*!< [in] Handle to open file */
) {
h5_int64_t nparticles;
ssize_t exists;
/* returns 0 if there are no datasets on disk */
TRY ( exists = h5_get_num_hdf5_datasets(f, f->step_gid ) );
if ( exists == 0 )
{
/* try to recover number of particles from a previous
* H5PartSetNumParticles call. */
#ifdef PARALLEL_IO
hsize_t total;
TRY( h5priv_mpi_sum(f,
&(f->u->nparticles), &total,
1, MPI_LONG_LONG, f->comm) );
nparticles = (h5_int64_t)total;
#else
nparticles = (h5_int64_t)f->u->nparticles;
#endif
if ( nparticles > 0 ) {
h5_debug (
f,
"Using existing view to report "
"nparticles = %lld", (long long)nparticles );
return nparticles;
}
else {
h5_warn (
f,
"There are no datasets in the current timestep "
"nor existing views: "
"reporting 0 particles.");
return 0;
}
}
/* if a view exists, use its size as the number of particles */
if ( h5u_has_view ( f ) )
@@ -47,38 +13,52 @@ h5u_get_num_particles (
TRY( nparticles = h5priv_get_selected_npoints_of_hdf5_dataspace(
f,
f->u->diskshape) );
h5_debug(
f,
"Found %lld points with H5Sget_select_npoints",
(long long)nparticles );
h5_debug(f,
"Found %lld particles in existing view.",
(long long)nparticles );
}
/* otherwise, report all particles on disk in the first dataset
* for this timestep */
else
{
else if ( f->u->shape > 0 ) {
TRY( nparticles = h5priv_get_npoints_of_hdf5_dataspace(f, f->u->shape) );
h5_debug(f,
"Found %lld particles from previous H5PartSetNumParticles call.",
(long long)nparticles );
}
else {
/* otherwise, report all particles on disk in the first dataset
* for this timestep */
char dataset_name[H5_DATANAME_LEN];
TRY( h5priv_get_hdf5_objname_by_idx(
dataset_name[0] = '\0';
h5_err_t exists = h5_get_hdf5_datasetname_by_idx(
f,
f->step_gid,
0,
dataset_name,
H5_DATANAME_LEN) );
TRY( nparticles = h5priv_get_npoints_of_hdf5_dataset_by_name (
H5_DATANAME_LEN);
if ( exists < 0 )
return h5_error(f,
H5_ERR_INVAL,
"Cannot determine the number of particles: "
"H5PartSetNumParticles has not been called, "
"no view has been set, and there are no "
"data sets for this time step!");
TRY( nparticles = h5priv_get_npoints_of_hdf5_dataset_by_name(
f,
f->step_gid,
dataset_name ) );
dataset_name) );
h5_debug(f,
"Found %lld particles in the first data set of this time step.",
(long long)nparticles );
}
return nparticles;
}
h5_int64_t
h5_err_t
h5u_set_num_particles (
h5_file_t *const f, /*!< [in] Handle to open file */
const h5_int64_t nparticles, /*!< [in] Number of particles */
const h5_int64_t stride /*!< [in] Stride of particles in memory */
const h5_size_t nparticles, /*!< [in] Number of particles */
const h5_size_t stride /*!< [in] Stride of particles in memory */
) {
CHECK_FILEHANDLE( f );
struct h5u_fdata *u = f->u;
hsize_t hstride;
@@ -88,22 +68,14 @@ h5u_set_num_particles (
hsize_t dmax = H5S_UNLIMITED;
if ( nparticles <= 0 )
return h5_error(
f,
return h5_error(f,
H5_ERR_INVAL,
"Invalid number particles: %lld!\n",
(long long)nparticles);
/* prevent invalid stride value */
if (stride < 1)
{
h5_warn (
f,
"Stride < 1 was specified: changing to 1." );
hstride = 1;
} else {
hstride = (hsize_t)stride;
}
hstride = (hsize_t)stride;
if ( hstride > 1 )
h5_debug( f, "Striding by %lld elements.", (long long)hstride);
#ifndef PARALLEL_IO
/*
@@ -135,22 +107,16 @@ h5u_set_num_particles (
{
start = 0;
count = u->nparticles;
TRY( h5priv_select_hyperslab_of_hdf5_dataspace(
f,
TRY( h5priv_select_hyperslab_of_hdf5_dataspace(f,
u->memshape,
H5S_SELECT_SET,
&start,
&hstride,
&count, NULL ) );
&start, &hstride, &count,
NULL) );
}
#ifndef PARALLEL_IO
count = u->nparticles;
TRY( u->shape = h5priv_create_hdf5_dataspace (
f,
1,
&count,
NULL ) );
TRY( u->shape = h5priv_create_hdf5_dataspace(f, 1, &count, NULL) );
u->viewstart = 0;
u->viewend = nparticles - 1; // view range is *inclusive*
#else /* PARALLEL_IO */
@@ -173,30 +139,33 @@ h5u_set_num_particles (
&(u->nparticles), &total, 1, MPI_LONG_LONG, f->comm ) );
TRY( h5priv_mpi_prefix_sum(f,
&(u->nparticles), &start, 1, MPI_LONG_LONG, f->comm ) );
start -= u->nparticles;
h5_debug(f, "Total particles across all processors: %lld.",
(long long)total);
u->viewstart = start;
u->viewend = start + u->nparticles - 1; // view range is *inclusive*
/* declare overall datasize */
count = total;
TRY ( u->shape = h5priv_create_hdf5_dataspace (f, 1, &count, NULL) );
TRY( u->shape = h5priv_create_hdf5_dataspace(f, 1, &count, NULL) );
/* declare overall data size but then will select a subset */
TRY ( u->diskshape = h5priv_create_hdf5_dataspace (f, 1, &count, NULL) );
TRY( u->diskshape = h5priv_create_hdf5_dataspace(f, 1, &count, NULL) );
count = nparticles;
hstride = 1;
TRY ( h5priv_select_hyperslab_of_hdf5_dataspace (
f,
TRY( h5priv_select_hyperslab_of_hdf5_dataspace(f,
u->diskshape,
H5S_SELECT_SET,
&start, &hstride, &count,
NULL ) );
NULL) );
#endif
return H5_SUCCESS;
}
h5_int64_t
h5_err_t
h5u_has_view (
const h5_file_t *const f
) {
@@ -264,8 +233,6 @@ h5u_set_view (
if ( start == -1 ) start = 0;
if ( end == -1 ) end = total - 1; // range is *inclusive*
h5_debug ( f, "Total nparticles=%lld", (long long)total );
/* so, is this selection inclusive or exclusive?
it appears to be inclusive for both ends of the range.
*/
@@ -282,7 +249,9 @@ h5u_set_view (
u->viewend = end;
u->nparticles = end - start + 1;
h5_debug ( f, "nparticles=%lld", (long long)u->nparticles );
h5_debug (f,
"This view selected %lld particles.",
(long long)u->nparticles );
/* declare overall data size but then will select a subset */
TRY ( u->diskshape = h5priv_create_hdf5_dataspace ( f, 1, &total, NULL ) );
@@ -304,11 +273,11 @@ h5u_set_view (
return H5_SUCCESS;
}
h5_int64_t
h5_err_t
h5u_set_view_indices (
h5_file_t *const f, /*!< [in] Handle to open file */
const h5_int64_t *const indices, /*!< [in] List of indices */
const h5_int64_t nelems /*!< [in] Size of list */
const h5_id_t *const indices, /*!< [in] List of indices */
const h5_size_t nelems /*!< [in] Size of list */
) {
hsize_t total;
@@ -338,19 +307,12 @@ h5u_set_view_indices (
return H5_SUCCESS;
}
h5_debug ( f, "Total nparticles=%lld", (long long)total );
if ( total == 0 ) return H5_SUCCESS;
/* check length of list */
if ( nelems < 0 ) {
h5_warn (f,
"Array of view indices has length < 0: "
"resetting view.");
u->nparticles = 0;
} else {
u->nparticles = (hsize_t) nelems;
}
u->nparticles = (hsize_t) nelems;
h5_debug (f,
"This view selected %lld particles.",
(long long)u->nparticles );
/* declare overall data size but then will select a subset */
TRY ( u->diskshape = h5priv_create_hdf5_dataspace ( f, 1, &total, NULL ) );
@@ -371,7 +333,7 @@ h5u_set_view_indices (
return H5_SUCCESS;
}
h5_int64_t
h5_err_t
h5u_get_view (
h5_file_t *const f,
h5_int64_t *start,
@@ -444,7 +406,7 @@ h5u_set_canonical_view (
return H5_SUCCESS;
}
h5_int64_t
h5_ssize_t
h5u_get_num_datasets (
h5_file_t *const f /*!< [in] Handle to open file */
) {
@@ -456,15 +418,15 @@ h5u_get_num_datasets (
/*!
Get information about dataset in current index given by its index
*/
h5_int64_t
h5_err_t
h5u_get_dataset_info (
h5_file_t *const f, /*!< [in] Handle to open file */
const h5_int64_t idx, /*!< [in] Index of the dataset */
const h5_id_t idx, /*!< [in] Index of the dataset */
char *dataset_name, /*!< [out] Name of dataset */
const h5_int64_t len_dataset_name,
const h5_size_t len_dataset_name,
/*!< [in] Size of buffer \c dataset_name */
h5_int64_t *type, /*!< [out] Type of data in dataset */
h5_int64_t *nelem /*!< [out] Number of elements. */
h5_size_t *nelem /*!< [out] Number of elements. */
) {
TRY( h5_get_hdf5_datasetname_by_idx (
src/h5core/h5u_readwrite.c
+4 -8
View File
@@ -9,6 +9,8 @@ h5u_read_data (
const hid_t type
) {
CHECK_TIMEGROUP( f );
struct h5u_fdata *u = f->u;
hid_t dataset_id;
hid_t space_id;
@@ -107,9 +109,9 @@ h5u_write_data (
const hid_t type /*!< IN: Type of data */
) {
CHECK_FILEHANDLE ( f );
CHECK_WRITABLE_MODE( f );
CHECK_TIMEGROUP( f );
CHECK_WRITABLE_MODE( f );
struct h5u_fdata *u = f->u;
hid_t dset_id;
@@ -138,9 +140,6 @@ h5u_write_data (
H5P_DEFAULT) );
}
#ifdef PARALLEL_IO
TRY( h5_start_throttle(f) );
#endif
h5_info (f,
"Writing dataset %s/%s.",
h5_get_objname(f->step_gid), name2);
@@ -153,9 +152,6 @@ h5u_write_data (
f->xfer_prop,
data) );
TRY( h5priv_close_hdf5_dataset (f, dset_id) );
#ifdef PARALLEL_IO
TRY( h5_end_throttle(f) );
#endif
f->empty = 0;