Add support for hdf5 int8 types (int8, uint8)

Resolve "hdf5 1.12 and newer support"

Closes #21

See merge request H5hut/src!6

configure.ac

@@ -6,10 +6,11 @@
 #
 #  License: see file COPYING in top level of source distribution.
 #
-AC_INIT([H5hut], [2.0.0rc6], [h5part@lists.psi.ch], H5hut)
+AC_INIT([H5hut], [2.0.0rc7], [h5part@lists.psi.ch], H5hut)
 AC_PREREQ(2.60)
 AC_CONFIG_HEADERS(config.h)
 AC_CONFIG_MACRO_DIR([m4])
+LT_INIT
 AM_INIT_AUTOMAKE
 
 ENABLE_DEBUG='no'

examples/eurohack24/read_setnparticles_ats.cu

@@ -0,0 +1,87 @@
+/*
+  Copyright (c) 2006-2015, The Regents of the University of California,
+  through Lawrence Berkeley National Laboratory (subject to receipt of any
+  required approvals from the U.S. Dept. of Energy) and the Paul Scherrer
+  Institut (Switzerland).  All rights reserved.
+
+  License: see file COPYING in top level of source distribution.
+*/
+
+#include "H5hut.h"
+#include "examples.h"
+
+#include <stdlib.h>
+
+#include "cuda.h"
+// name of input file
+const char* fname = "example_setnparticles.h5";
+
+// H5hut verbosity level
+const h5_int64_t h5_verbosity = H5_VERBOSE_DEFAULT;
+
+// #define USE_CUDA_KERNEL
+#ifdef USE_CUDA_KERNEL
+__global__
+#endif
+void kernel(h5_int32_t *data, h5_size_t n)
+{
+	for (h5_size_t i=0; i<n; i++) {
+                data[i] += 2;
+	}
+}
+
+
+int
+main (
+        int argc, char* argv[]
+        ){
+
+        // initialize MPI & H5hut
+        MPI_Init (&argc, &argv);
+        MPI_Comm comm = MPI_COMM_WORLD;
+        int comm_size = 1;
+        MPI_Comm_size (comm, &comm_size);
+        int comm_rank = 0;
+        MPI_Comm_rank (comm, &comm_rank);
+        H5AbortOnError ();
+        H5SetVerbosityLevel (h5_verbosity);
+
+        // open file and go to first step
+        h5_file_t file = H5OpenFile (fname, H5_O_RDONLY, H5_PROP_DEFAULT);
+        H5SetStep (file, 0);
+
+        // compute number of particles this process has to read
+        h5_ssize_t num_particles_total = H5PartGetNumParticles (file);
+        h5_ssize_t num_particles = num_particles_total / comm_size;
+        if (comm_rank+1 == comm_size)
+                num_particles += num_particles_total % comm_size;
+
+	printf ("[proc %d]: particles in view: %lld\n", comm_rank, (long long)num_particles);
+	printf ("[proc %d]: total number of particles: %lld\n",
+		comm_rank, (long long unsigned)num_particles_total);
+
+	// set number of particles
+        H5PartSetNumParticles (file, num_particles);
+
+        // read and print data
+        h5_int32_t* data = (h5_int32_t*)calloc (num_particles, sizeof (*data));
+        H5PartReadDataInt32 (file, "data", data);
+        H5CloseFile (file);
+
+#ifdef USE_CUDA_KERNEL
+	kernel<<<1, 1>>>(data, num_particles);
+#else
+	kernel(data, num_particles);
+#endif
+	int ec=cudaDeviceSynchronize();
+	printf("%d\n", ec);
+	for (int i = 0; i < num_particles; i++) {
+                printf ("[proc %d]: local index = %d, value = %d\n",
+                        comm_rank, i, data[i]);
+        }
+
+        // cleanup
+	free (data);
+	MPI_Finalize ();
+        return 0;
+}

examples/eurohack24/read_setnparticles_ats.sbatch

@@ -0,0 +1,14 @@
+#!/bin/bash
+#SBATCH --uenv=eurohack/24.9:rc1
+#SBATCH --view=modules
+#SBATCH --ntasks-per-node=1
+#SBATCH --nodes=1
+#SBATCH --output=out-%j.out
+#SBATCH -C gpu
+#SBATCH --partition=debug
+#SBATCH --time=00:05:00
+#
+export NSYS_NVTX_PROFILER_REGISTER_ONLY=0
+export CUDA_LAUNCH_BLOCKING=1
+EXE="${HOME}/src/H5hut/src/examples/H5Part/read_setnparticles_ats"
+nsys profile -t cuda,nvtx,mpi -o report.%p $EXE

examples/eurohack24/read_setnparticles_managed.cu

@@ -0,0 +1,96 @@
+/*
+  Copyright (c) 2006-2015, The Regents of the University of California,
+  through Lawrence Berkeley National Laboratory (subject to receipt of any
+  required approvals from the U.S. Dept. of Energy) and the Paul Scherrer
+  Institut (Switzerland).  All rights reserved.
+
+  License: see file COPYING in top level of source distribution.
+*/
+
+#include "H5hut.h"
+#include "examples.h"
+
+#include <stdlib.h>
+
+#include "cuda.h"
+// name of input file
+const char* fname = "example_setnparticles.h5";
+
+// H5hut verbosity level
+const h5_int64_t h5_verbosity = H5_VERBOSE_DEFAULT;
+
+#define USE_CUDA_KERNEL 1
+#ifdef USE_CUDA_KERNEL
+__global__
+#endif
+void kernel(h5_int32_t *data, h5_size_t n)
+{
+	for (h5_size_t i=0; i<n; i++) {
+                data[i] += 2;
+	}
+}
+
+
+int
+main (
+        int argc, char* argv[]
+        ){
+
+        // initialize MPI & H5hut
+        MPI_Init (&argc, &argv);
+        MPI_Comm comm = MPI_COMM_WORLD;
+        int comm_size = 1;
+        MPI_Comm_size (comm, &comm_size);
+        int comm_rank = 0;
+        MPI_Comm_rank (comm, &comm_rank);
+        H5AbortOnError ();
+        H5SetVerbosityLevel (h5_verbosity);
+
+        // open file and go to first step
+        h5_file_t file = H5OpenFile (fname, H5_O_RDONLY, H5_PROP_DEFAULT);
+        H5SetStep (file, 0);
+
+        // compute number of particles this process has to read
+        h5_ssize_t num_particles_total = H5PartGetNumParticles (file);
+        h5_ssize_t num_particles = num_particles_total / comm_size;
+        if (comm_rank+1 == comm_size)
+                num_particles += num_particles_total % comm_size;
+
+	printf ("[proc %d]: particles in view: %lld\n", comm_rank, (long long)num_particles);
+	printf ("[proc %d]: total number of particles: %lld\n",
+		comm_rank, (long long unsigned)num_particles_total);
+
+	// set number of particles
+        H5PartSetNumParticles (file, num_particles);
+
+        // read and print data
+	h5_int32_t *data;
+#ifdef USE_CUDA_KERNEL
+	cudaMallocManaged((void **)&data, num_particles * sizeof(*data));
+#else
+	data = (h5_int32_t*)calloc (num_particles, sizeof (*data));
+#endif
+	H5PartReadDataInt32 (file, "data", data);
+        H5CloseFile (file);
+
+#ifdef USE_CUDA_KERNEL
+	kernel<<<1, 1>>>(data, num_particles);
+#else
+	kernel(data, num_particles);
+#endif
+	int ec=cudaDeviceSynchronize();
+	printf("%d\n", ec);
+	for (int i = 0; i < num_particles; i++) {
+                printf ("[proc %d]: local index = %d, value = %d\n",
+                        comm_rank, i, data[i]);
+        }
+
+        // cleanup
+#ifdef USE_CUDA_KERNEL
+	cudaFree(data);
+#else
+	free (data);
+#endif
+	MPI_Finalize ();
+        return 0;
+}

examples/eurohack24/setnparticule.sbatch

@@ -0,0 +1,15 @@
+#!/bin/bash
+#SBATCH --uenv=eurohack/24.9:rc1
+#SBATCH --view=modules
+#SBATCH --ntasks-per-node=1
+#SBATCH --nodes=1
+#SBATCH --output=out-%j.out
+#SBATCH -C gpu
+#SBATCH --time=00:05:00
+#SBATCH --reservation=eurohack24
+#
+export CUDA_LAUNCH_BLOCKING=1
+#
+
+#nsys profile -t cuda,mpi -o report.%p read_setnparticles_managed
+ncu --kernel-name kernel --launch-skip 0 --launch-count 1 -o report.%p "read_setnparticles_managed"

examples/eurohack24/write_setnparticles.sbatch

@@ -0,0 +1,15 @@
+#!/bin/bash
+#SBATCH --uenv=eurohack/24.9:rc1
+#SBATCH --view=modules
+#SBATCH --ntasks-per-node=16
+#SBATCH --nodes=4
+#SBATCH --output=out-%j.out
+#SBATCH -C gpu
+#SBATCH --partition=debug
+#SBATCH --time=00:05:00
+#
+export NSYS_NVTX_PROFILER_REGISTER_ONLY=0
+export CUDA_LAUNCH_BLOCKING=1
+EXE="${HOME}/src/H5hut/src/examples/H5Part/write_setnparticles"
+srun -n 64  "$EXE"
+#nsys profile -t cuda,nvtx,mpi -o report.%p $EXE

src/h5core/h5_file.c

@@ -92,7 +92,7 @@ mpi_init (
 		TRY (hdf5_set_fapl_mpiposix_property (f->props->access_prop,
                                                      f->props->comm, use_gpfs));
 
-        } else if ((f->props->flags & H5_VFD_CORE)) {
+        } else if ((f->props->flags & H5_VFD_CORE_IO)) {
 		h5_info("Selecting CORE VFD");
                 TRY (hdf5_set_fapl_core (f->props->access_prop,
                                          f->props->align, 1));
@@ -112,7 +112,7 @@ mpi_init (
 	}
 #else
 	// VFD_MPIO_POSIX has been removed in HDF5 1.8.13
-        if ((f->props->flags & H5_VFD_CORE)) {
+        if ((f->props->flags & H5_VFD_CORE_IO)) {
 		h5_info("Selecting CORE VFD");
                 TRY (hdf5_set_fapl_core (f->props->access_prop,
                                          f->props->align, 1));
@@ -197,7 +197,7 @@ h5_set_prop_file_mpio_collective (
 			(long long int)props->class);
         }
 #ifdef H5_HAVE_PARALLEL
-        props->flags &= ~(H5_VFD_MPIO_POSIX | H5_VFD_MPIO_INDEPENDENT | H5_VFD_CORE);
+        props->flags &= ~(H5_VFD_MPIO_POSIX | H5_VFD_MPIO_INDEPENDENT | H5_VFD_CORE_IO);
         props->flags |= H5_VFD_MPIO_COLLECTIVE;
         props->comm = *comm;
 	if (props->throttle > 0) {
@@ -225,7 +225,7 @@ h5_set_prop_file_mpio_independent (
 			(long long int)props->class);
         }
 #ifdef H5_HAVE_PARALLEL
-        props->flags &= ~(H5_VFD_MPIO_COLLECTIVE | H5_VFD_MPIO_POSIX | H5_VFD_CORE);
+        props->flags &= ~(H5_VFD_MPIO_COLLECTIVE | H5_VFD_MPIO_POSIX | H5_VFD_CORE_IO);
         props->flags |= H5_VFD_MPIO_INDEPENDENT;
         props->comm = *comm;
 #else
@@ -250,7 +250,7 @@ h5_set_prop_file_mpio_posix (
 			(long long int)props->class);
         }
 #ifdef H5_HAVE_PARALLEL
-        props->flags &= ~(H5_VFD_MPIO_COLLECTIVE | H5_VFD_MPIO_POSIX | H5_VFD_CORE);
+        props->flags &= ~(H5_VFD_MPIO_COLLECTIVE | H5_VFD_MPIO_POSIX | H5_VFD_CORE_IO);
         props->flags |= H5_VFD_MPIO_INDEPENDENT;
         props->comm = *comm;
 #else

src/h5core/private/h5_attribs.h

@@ -99,7 +99,7 @@ h5priv_write_attrib (
 	                   attrib_name,
 	                   (long long int)attrib_type,
 	                   attrib_value,
-	                   attrib_nelem);
+	                   (long long unsigned)attrib_nelem);
 	hid_t space_id;
 	hid_t attrib_id;
 	hid_t hdf5_type;
@@ -147,7 +147,7 @@ h5priv_append_attrib (
 	                   attrib_name,
 	                   (long long int)attrib_type,
 	                   attrib_value,
-	                   attrib_nelem);
+	                   (long long unsigned)attrib_nelem);
 	h5_err_t exists;
 	TRY (exists = hdf5_attribute_exists (id, attrib_name));
 	if (exists) {

src/h5core/private/h5_file.h

@@ -9,7 +9,7 @@
 #define H5_VFD_MPIO_POSIX       0x00000010
 #define H5_VFD_MPIO_INDEPENDENT 0x00000020
 #define H5_VFD_MPIO_COLLECTIVE  0x00000040
-#define H5_VFD_CORE		0x00000080
+#define H5_VFD_CORE_IO		0x00000080
 
 #define H5_FLUSH_FILE		0x00001000
 #define H5_FLUSH_ITERATION	0x00002000

src/h5core/private/h5_hdf5.c

@@ -110,10 +110,18 @@ iter_op_get_obj_type (
 			        name);
 			return H5O_TYPE_UNKNOWN;
 		}
+#if H5_VERSION_GE(1,12,0)
+		herr = H5Oget_info(obj_id, &objinfo, H5O_INFO_ALL);
+#else
 		herr = H5Oget_info(obj_id, &objinfo);
+#endif
 	}
 	else { // H5L_TYPE_HARD
+#if H5_VERSION_GE(1,12,0)
+	  herr = H5Oget_info_by_name(g_id, name, &objinfo, H5O_INFO_BASIC, H5P_DEFAULT);
+#else
 		herr = H5Oget_info_by_name(g_id, name, &objinfo, H5P_DEFAULT);
+#endif
 	}
 
 	if (herr < 0) {
@@ -245,7 +253,8 @@ hdf5_get_name_of_group_by_idx (
 	HDF5_WRAPPER_ENTER (h5_err_t,
 	                    "loc_id=%lld (%s), idx=%llu, name=%p, len=%llu",
 	                    (long long int)loc_id, hdf5_get_objname (loc_id),
-	                    idx, name, (unsigned long long)len);
+	                    (long long unsigned)idx,
+			    name, (unsigned long long)len);
 	op_data_t op_data;
 	memset (&op_data, 0, sizeof (op_data));
 	op_data.type = H5O_TYPE_GROUP;
@@ -304,7 +313,8 @@ hdf5_get_name_of_dataset_by_idx (
 	HDF5_WRAPPER_ENTER (h5_err_t,
 	                    "loc_id=%lld (%s), idx=%llu, name=%p, len=%llu",
 	                    (long long int)loc_id, hdf5_get_objname (loc_id),
-	                    idx, name, (unsigned long long)len);
+	                    (long long unsigned)idx,
+			    name, (unsigned long long)len);
 	op_data_t op_data;
 	memset (&op_data, 0, sizeof (op_data));
 	op_data.type = H5O_TYPE_DATASET;

src/h5core/private/h5_hdf5.h

@@ -718,7 +718,7 @@ hdf5_set_dataset_extent (
 	                    "dataset_id=%lld (%s), size=%llu",
 	                    (long long int)dataset_id,
 	                    hdf5_get_objname(dataset_id),
-	                    *size);
+	                    (long long unsigned int)*size);
 	if (H5Dset_extent(dataset_id, size) < 0) {
 		H5_RETURN_ERROR (
 			H5_ERR_HDF5,
@@ -873,7 +873,7 @@ static inline hid_t
 hdf5_create_string_type(
         const hsize_t len
         ) {
-	HDF5_WRAPPER_ENTER (hid_t, "len = %llu", len);
+  HDF5_WRAPPER_ENTER (hid_t, "len = %llu", (long long unsigned)len);
 	hid_t type_id = H5Tcopy (H5T_C_S1);
 	if (type_id < 0)
 		H5_RETURN_ERROR (
@@ -1003,7 +1003,7 @@ hdf5_set_chunk_property (
         ) {
 	HDF5_WRAPPER_ENTER (h5_err_t,
 	                    "plist=%lld, rank=%d, dims[0]=%llu ...",
-	                    (long long int)plist, rank, dims[0]);
+	                    (long long int)plist, rank, (long long unsigned)dims[0]);
 	if (H5Pset_chunk (plist, rank, dims) < 0)
 		H5_RETURN_ERROR (
 			H5_ERR_HDF5,
@@ -1146,7 +1146,8 @@ hdf5_set_btree_ik_property (
         ) {
 	HDF5_WRAPPER_ENTER (h5_err_t,
 	                    "fapl_id=%lld, btree_ik=%llu",
-	                    (long long int)fcpl_id, btree_ik);
+	                    (long long int)fcpl_id,
+			    (long long unsigned)btree_ik);
 	if (H5Pset_istore_k (fcpl_id, btree_ik) < 0)
 		H5_RETURN_ERROR (
 			H5_ERR_HDF5,
@@ -1164,13 +1165,16 @@ hdf5_set_alignment_property (
         ) {
 	HDF5_WRAPPER_ENTER (h5_err_t,
 	                    "plist=%lld, threshold=%llu, alignment=%llu",
-	                    (long long int)plist, threshold, alignment);
+	                    (long long int)plist, 
+			    (long long unsigned)threshold,
+			    (long long unsigned)alignment);
 	if (H5Pset_alignment (plist, threshold, alignment) < 0)
 		H5_RETURN_ERROR (
 			H5_ERR_HDF5,
 			"Cannot set alignment property to %llu "
 			"and threshold %llu",
-			alignment, threshold);
+			(long long unsigned)alignment,
+			(long long unsigned)threshold);
 	H5_RETURN (H5_SUCCESS);
 }
 
@@ -1181,12 +1185,13 @@ hdf5_set_meta_block_size (
         ) {
 	HDF5_WRAPPER_ENTER (h5_err_t,
 	                    "fapl_id=%lld, size=%llu",
-	                    (long long int)fapl_id, size);
+	                    (long long int)fapl_id,
+			    (long long unsigned)size);
 	if (H5Pset_meta_block_size (fapl_id, size) < 0)
 		H5_RETURN_ERROR (
 			H5_ERR_HDF5,
 			"Cannot set meta block size property to %llu",
-			size);
+			(long long unsigned)size);
 	H5_RETURN (H5_SUCCESS);
 }
 
@@ -1292,9 +1297,15 @@ hdf5_close_file (
 		for (ssize_t i = 0; i < max_objs; i++) {
 			hid_t object_id = obj_id_list [i];
 			h5_debug ("Open object: %lld", (long long)object_id);
+#if H5_VERSION_GE(1,12,0)
+			H5O_info_t object_info;
+			if (H5Oget_info (object_id, &object_info, H5O_INFO_ALL) < 0)
+				continue;
+#else
 			H5O_info_t object_info;
 			if (H5Oget_info (object_id, &object_info) < 0)
 				continue;
+#endif
 			switch (object_info.type) {
 			case H5O_TYPE_GROUP:
 			case H5O_TYPE_DATASET:

src/h5core/private/h5_model.h

@@ -135,6 +135,12 @@ h5priv_map_enum_to_normalized_type (
 	case H5_STRING_T:
 		ret_value = H5_STRING;
 		break;
+	case H5_INT8_T: 
+		ret_value = H5_INT8;
+		break;
+	case H5_UINT8_T:
+		ret_value = H5_UINT8;
+		break;
 	case H5_INT16_T:
 		ret_value = H5_INT16;
 		break;
@@ -203,6 +209,12 @@ h5priv_normalize_type (
 			} else {
 				ret_value = H5_UINT16;
 			}
+		} else if (tsize==1) {
+			if (tsign == H5T_SGN_2) {
+				ret_value = H5_INT8;
+			} else {
+				ret_value = H5_UINT8;
+			}
 		}
 		break;
 	case H5T_FLOAT:
@@ -263,6 +275,13 @@ h5priv_map_hdf5_type_to_enum (
 			} else {
 				ret_value = H5_UINT16_T;
 			}
+		} else if (tsize==1) {
+			if (tsign == H5T_SGN_2) {
+				ret_value = H5_INT8_T;
+			}
+			else {
+				ret_value = H5_UINT8_T;
+			}
 		} else {
 			ret_value = H5_STRING_T;
 		}

src/h5core/private/h5_types.h

@@ -13,6 +13,8 @@
 #include <hdf5.h>
 #include "h5core/h5_types.h"
 
+#define H5_INT8			H5T_NATIVE_INT8
+#define H5_UINT8		H5T_NATIVE_UINT8
 #define H5_INT16                H5T_NATIVE_INT16
 #define H5_UINT16		H5T_NATIVE_UINT16
 #define H5_INT32                H5T_NATIVE_INT32

src/include/H5_const.f90

@@ -13,7 +13,8 @@
 
   INTEGER*8, PARAMETER :: H5_VFD_MPIPOSIX =        Z'00000010'
   INTEGER*8, PARAMETER :: H5_VFD_MPIIO_IND =       Z'00000020'
-  INTEGER*8, PARAMETER :: H5_VFD_CORE =            Z'00000040'
+  INTEGER*8, PARAMETER :: H5_VFD_MPIIO_COL =       Z'00000040'
+  INTEGER*8, PARAMETER :: H5_VFD_CORE_IO =         Z'00000080'
 
   INTEGER*8, PARAMETER :: H5_PROP_DEFAULT =        0
 

src/include/h5core/h5_types.h

@@ -30,6 +30,8 @@ typedef int MPI_Datatype;
 
 typedef enum  {
 	H5_STRING_T,
+	H5_INT8_T,
+	H5_UINT8_T,
 	H5_INT16_T,
 	H5_UINT16_T,
 	H5_INT32_T,

test/testframe.c

@@ -658,7 +658,11 @@ test_open_objects(h5_file_t file, int max_objects)
 		H5O_info_t info;
 		int i;
 		for (i=0; i<nopen; i++) {
-			H5Oget_info(list[i], &info);
+#if H5_VERSION_GE(1,12,0)
+		  H5Oget_info(list[i], &info, H5O_INFO_BASIC);
+#else
+		  H5Oget_info(list[i], &info);
+#endif
 			switch (info.type) {
 			case H5O_TYPE_GROUP:
 				TestErrPrintf("obj%d has type GROUP\n", i);