Files
Jungfraujoch/writer/H5FDpoison_sec2.c
T
leonarski_f caef26873e
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 16m26s
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 14m26s
Build Packages / build:rpm (rocky8) (push) Successful in 17m23s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 17m32s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 18m16s
Build Packages / build:rpm (rocky9) (push) Successful in 12m45s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 12m58s
Build Packages / XDS test (durin plugin) (push) Successful in 11m22s
Build Packages / DIALS test (push) Successful in 14m28s
Build Packages / Generate python client (push) Successful in 1m1s
Build Packages / Build documentation (push) Successful in 2m40s
Build Packages / Create release (push) Has been skipped
Build Packages / XDS test (neggia plugin) (push) Successful in 10m52s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 15m2s
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 17m25s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 11m49s
Build Packages / XDS test (JFJoch plugin) (push) Successful in 11m34s
Build Packages / Unit tests (push) Successful in 44m51s
v1.0.0-rc.145 (#55)
This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.
* XDS plugin: Improve performance of VDS reading.
* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.
* jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

Reviewed-on: #55
2026-05-06 21:50:02 +02:00

681 lines
18 KiB
C

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only OR HDF5
// This file may be used, modified, and distributed under either GPL-3.0-only
// or the HDF5 license, at the recipient's option.
#include "H5FDpoison_sec2.h"
#include <errno.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <hdf5.h>
#include <H5FDdevelop.h>
#define H5FD_POISON_SEC2_MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1)
#ifndef H5FD_POISON_SEC2_NAME
#define H5FD_POISON_SEC2_NAME "poison_sec2"
#endif
#ifndef H5FD_POISON_SEC2_VALUE
#define H5FD_POISON_SEC2_VALUE ((H5FD_class_value_t)(H5_VFD_RESERVED + 42))
#endif
#define H5FD_POISON_SEC2_LAST_ERROR_STRING_SIZE 1024
typedef struct H5FD_poison_sec2_t {
H5FD_t pub;
H5FD_t *inner;
hbool_t poisoned;
int poison_errno;
char *filename;
} H5FD_poison_sec2_t;
typedef struct H5FD_poison_sec2_last_error_t {
hbool_t valid;
int error_number;
char filename[H5FD_POISON_SEC2_LAST_ERROR_STRING_SIZE];
char operation[H5FD_POISON_SEC2_LAST_ERROR_STRING_SIZE];
} H5FD_poison_sec2_last_error_t;
static hid_t H5FD_POISON_SEC2_id_g = H5I_INVALID_HID;
static H5FD_poison_sec2_cb_t H5FD_poison_sec2_cb_g = NULL;
static void *H5FD_poison_sec2_cb_ud_g = NULL;
static hbool_t H5FD_poison_sec2_fail_pass_g = false;
static H5FD_poison_sec2_last_error_t H5FD_poison_sec2_last_error_g = {
false,
0,
"",
""
};
static void H5FD_poison_sec2_clear_last_error(void) {
H5FD_poison_sec2_last_error_g.valid = false;
H5FD_poison_sec2_last_error_g.error_number = 0;
H5FD_poison_sec2_last_error_g.filename[0] = '\0';
H5FD_poison_sec2_last_error_g.operation[0] = '\0';
}
static void H5FD_poison_sec2_store_last_error(
const H5FD_poison_sec2_t *file,
const char *operation,
int error_number
) {
const char *filename = file != NULL && file->filename != NULL ? file->filename : "";
const char *op = operation != NULL ? operation : "";
H5FD_poison_sec2_last_error_g.valid = true;
H5FD_poison_sec2_last_error_g.error_number = error_number != 0 ? error_number : EIO;
strncpy(
H5FD_poison_sec2_last_error_g.filename,
filename,
sizeof(H5FD_poison_sec2_last_error_g.filename) - 1
);
H5FD_poison_sec2_last_error_g.filename[
sizeof(H5FD_poison_sec2_last_error_g.filename) - 1
] = '\0';
strncpy(
H5FD_poison_sec2_last_error_g.operation,
op,
sizeof(H5FD_poison_sec2_last_error_g.operation) - 1
);
H5FD_poison_sec2_last_error_g.operation[
sizeof(H5FD_poison_sec2_last_error_g.operation) - 1
] = '\0';
}
static herr_t H5FD_poison_sec2_fail_fast(H5FD_poison_sec2_t *file, const char *operation) {
if (!file->poisoned)
return 0;
errno = file->poison_errno != 0 ? file->poison_errno : EIO;
H5FD_poison_sec2_store_last_error(file, operation, errno);
return -1;
}
static herr_t H5FD_poison_sec2_mark_poisoned(
H5FD_poison_sec2_t *file,
const char *operation
) {
if (!file->poisoned) {
file->poisoned = true;
file->poison_errno = errno != 0 ? errno : EIO;
if (H5FD_poison_sec2_cb_g != NULL) {
H5FD_poison_sec2_cb_g(
file->filename != NULL ? file->filename : "",
operation,
file->poison_errno,
H5FD_poison_sec2_cb_ud_g
);
}
} else if (errno != 0) {
file->poison_errno = errno;
}
H5FD_poison_sec2_store_last_error(file, operation, file->poison_errno);
errno = file->poison_errno != 0 ? file->poison_errno : EIO;
return -1;
}
static herr_t H5FD_poison_sec2_write_failure_result(void) {
return H5FD_poison_sec2_fail_pass_g ? 0 : -1;
}
static hid_t H5FD_poison_sec2_make_inner_fapl(hid_t fapl_id) {
hid_t inner_fapl = H5I_INVALID_HID;
/*
* Copy the caller's FAPL so generic file-access settings are preserved:
* alignment, metadata block size, sieve buffer size, file locking,
* libver bounds, close degree, etc.
*
* Then replace only the VFD driver with sec2. This prevents recursive
* poison_sec2 -> poison_sec2 opens while keeping the rest of the FAPL.
*/
if (fapl_id == H5P_DEFAULT)
inner_fapl = H5Pcreate(H5P_FILE_ACCESS);
else
inner_fapl = H5Pcopy(fapl_id);
if (inner_fapl < 0)
return H5I_INVALID_HID;
if (H5Pset_fapl_sec2(inner_fapl) < 0) {
H5Pclose(inner_fapl);
return H5I_INVALID_HID;
}
return inner_fapl;
}
static H5FD_t *H5FD_poison_sec2_open(
const char *name,
unsigned flags,
hid_t fapl_id,
haddr_t maxaddr
) {
H5FD_poison_sec2_t *file = NULL;
hid_t inner_fapl = H5I_INVALID_HID;
file = (H5FD_poison_sec2_t *)calloc(1, sizeof(H5FD_poison_sec2_t));
if (file == NULL)
return NULL;
file->inner = NULL;
file->poisoned = false;
file->poison_errno = 0;
file->filename = name != NULL ? strdup(name) : NULL;
inner_fapl = H5FD_poison_sec2_make_inner_fapl(fapl_id);
if (inner_fapl < 0) {
free(file->filename);
free(file);
return NULL;
}
file->inner = H5FDopen(name, flags, inner_fapl, maxaddr);
H5Pclose(inner_fapl);
if (file->inner == NULL) {
free(file->filename);
free(file);
return NULL;
}
return (H5FD_t *)file;
}
static herr_t H5FD_poison_sec2_close(H5FD_t *_file) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner != NULL) {
herr_t err = 0;
H5E_BEGIN_TRY {
err = H5FDclose(file->inner);
} H5E_END_TRY;
if (err < 0)
H5FD_poison_sec2_mark_poisoned(file, "close");
file->inner = NULL;
}
free(file->filename);
free(file);
return 0;
}
static int H5FD_poison_sec2_cmp(const H5FD_t *_f1, const H5FD_t *_f2) {
const H5FD_poison_sec2_t *f1 = (const H5FD_poison_sec2_t *)_f1;
const H5FD_poison_sec2_t *f2 = (const H5FD_poison_sec2_t *)_f2;
if (f1->inner == NULL && f2->inner == NULL)
return 0;
if (f1->inner == NULL)
return -1;
if (f2->inner == NULL)
return 1;
return H5FDcmp(f1->inner, f2->inner);
}
static herr_t H5FD_poison_sec2_query(const H5FD_t *_file, unsigned long *flags) {
(void)_file;
if (flags == NULL)
return -1;
*flags = H5FD_FEAT_AGGREGATE_METADATA |
H5FD_FEAT_ACCUMULATE_METADATA |
H5FD_FEAT_DATA_SIEVE |
H5FD_FEAT_AGGREGATE_SMALLDATA;
return 0;
}
static haddr_t H5FD_poison_sec2_get_eoa(const H5FD_t *_file, H5FD_mem_t type) {
const H5FD_poison_sec2_t *file = (const H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return HADDR_UNDEF;
return H5FDget_eoa(file->inner, type);
}
static herr_t H5FD_poison_sec2_set_eoa(
H5FD_t *_file,
H5FD_mem_t type,
haddr_t addr
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
/*
* EOA is logical state, not necessarily physical I/O.
* Do not fail-fast this unless inner sec2 itself rejects it.
*/
if (H5FDset_eoa(file->inner, type, addr) < 0)
return H5FD_poison_sec2_mark_poisoned(file, "set_eoa");
return 0;
}
static haddr_t H5FD_poison_sec2_get_eof(const H5FD_t *_file, H5FD_mem_t type) {
const H5FD_poison_sec2_t *file = (const H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return HADDR_UNDEF;
return H5FDget_eof(file->inner, type);
}
static herr_t H5FD_poison_sec2_get_handle(
H5FD_t *_file,
hid_t fapl,
void **file_handle
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
/*
* Preserve sec2 semantics: return the native handle from inner sec2.
*/
return H5FDget_vfd_handle(file->inner, fapl, file_handle);
}
static herr_t H5FD_poison_sec2_read(
H5FD_t *_file,
H5FD_mem_t type,
hid_t dxpl_id,
haddr_t addr,
size_t size,
void *buf
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "read") < 0)
return -1;
return H5FDread(file->inner, type, dxpl_id, addr, size, buf);
}
static herr_t H5FD_poison_sec2_write(
H5FD_t *_file,
H5FD_mem_t type,
hid_t dxpl_id,
haddr_t addr,
size_t size,
const void *buf
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "write") < 0)
return H5FD_poison_sec2_write_failure_result();
if (H5FDwrite(file->inner, type, dxpl_id, addr, size, buf) < 0) {
H5FD_poison_sec2_mark_poisoned(file, "write");
return H5FD_poison_sec2_write_failure_result();
}
return 0;
}
static herr_t H5FD_poison_sec2_read_vector(
H5FD_t *_file,
hid_t dxpl_id,
uint32_t count,
H5FD_mem_t types[],
haddr_t addrs[],
size_t sizes[],
void *bufs[]
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "read_vector") < 0)
return -1;
return H5FDread_vector(
file->inner,
dxpl_id,
count,
types,
addrs,
sizes,
bufs
);
}
static herr_t H5FD_poison_sec2_write_vector(
H5FD_t *_file,
hid_t dxpl_id,
uint32_t count,
H5FD_mem_t types[],
haddr_t addrs[],
size_t sizes[],
const void *bufs[]
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "write_vector") < 0)
return H5FD_poison_sec2_write_failure_result();
if (H5FDwrite_vector(
file->inner,
dxpl_id,
count,
types,
addrs,
sizes,
bufs
) < 0) {
H5FD_poison_sec2_mark_poisoned(file, "write_vector");
return H5FD_poison_sec2_write_failure_result();
}
return 0;
}
static herr_t H5FD_poison_sec2_read_selection(
H5FD_t *_file,
H5FD_mem_t type,
hid_t dxpl_id,
size_t count,
hid_t mem_spaces[],
hid_t file_spaces[],
haddr_t offsets[],
size_t element_sizes[],
void *bufs[]
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "read_selection") < 0)
return -1;
return H5FDread_selection(
file->inner,
type,
dxpl_id,
(uint32_t)count,
mem_spaces,
file_spaces,
offsets,
element_sizes,
bufs
);
}
static herr_t H5FD_poison_sec2_write_selection(
H5FD_t *_file,
H5FD_mem_t type,
hid_t dxpl_id,
size_t count,
hid_t mem_spaces[],
hid_t file_spaces[],
haddr_t offsets[],
size_t element_sizes[],
const void *bufs[]
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "write_selection") < 0)
return H5FD_poison_sec2_write_failure_result();
if (H5FDwrite_selection(
file->inner,
type,
dxpl_id,
(uint32_t)count,
mem_spaces,
file_spaces,
offsets,
element_sizes,
bufs
) < 0) {
H5FD_poison_sec2_mark_poisoned(file, "write_selection");
return H5FD_poison_sec2_write_failure_result();
}
return 0;
}
static herr_t H5FD_poison_sec2_flush(
H5FD_t *_file,
hid_t dxpl_id,
bool closing
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "flush") < 0)
return H5FD_poison_sec2_write_failure_result();
if (H5FDflush(file->inner, dxpl_id, closing) < 0) {
H5FD_poison_sec2_mark_poisoned(file, "flush");
return H5FD_poison_sec2_write_failure_result();
}
return 0;
}
static herr_t H5FD_poison_sec2_truncate(
H5FD_t *_file,
hid_t dxpl_id,
bool closing
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
if (H5FD_poison_sec2_fail_fast(file, "truncate") < 0)
return H5FD_poison_sec2_write_failure_result();
if (H5FDtruncate(file->inner, dxpl_id, closing) < 0) {
H5FD_poison_sec2_mark_poisoned(file, "truncate");
return H5FD_poison_sec2_write_failure_result();
}
return 0;
}
static herr_t H5FD_poison_sec2_lock(H5FD_t *_file, bool rw) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
return H5FDlock(file->inner, rw);
}
static herr_t H5FD_poison_sec2_unlock(H5FD_t *_file) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file->inner == NULL)
return -1;
return H5FDunlock(file->inner);
}
static herr_t H5FD_poison_sec2_delete(const char *filename, hid_t fapl_id) {
hid_t inner_fapl = H5FD_poison_sec2_make_inner_fapl(fapl_id);
herr_t ret;
if (inner_fapl < 0)
return -1;
ret = H5FDdelete(filename, inner_fapl);
H5Pclose(inner_fapl);
return ret;
}
static herr_t H5FD_poison_sec2_ctl(
H5FD_t *_file,
uint64_t op_code,
uint64_t flags,
const void *input,
void **output
) {
H5FD_poison_sec2_t *file = (H5FD_poison_sec2_t *)_file;
if (file == NULL || file->inner == NULL)
return -1;
return H5FDctl(file->inner, op_code, flags, input, output);
}
static const H5FD_class_t H5FD_poison_sec2_class_g = {
H5FD_CLASS_VERSION, /* struct version */
H5FD_POISON_SEC2_VALUE, /* value */
H5FD_POISON_SEC2_NAME, /* name */
H5FD_POISON_SEC2_MAXADDR, /* maxaddr */
H5F_CLOSE_WEAK, /* fc_degree */
NULL, /* terminate */
NULL, /* sb_size */
NULL, /* sb_encode */
NULL, /* sb_decode */
0, /* fapl_size */
NULL, /* fapl_get */
NULL, /* fapl_copy */
NULL, /* fapl_free */
0, /* dxpl_size */
NULL, /* dxpl_copy */
NULL, /* dxpl_free */
H5FD_poison_sec2_open, /* open */
H5FD_poison_sec2_close, /* close */
H5FD_poison_sec2_cmp, /* cmp */
H5FD_poison_sec2_query, /* query */
NULL, /* get_type_map */
NULL, /* alloc */
NULL, /* free */
H5FD_poison_sec2_get_eoa, /* get_eoa */
H5FD_poison_sec2_set_eoa, /* set_eoa */
H5FD_poison_sec2_get_eof, /* get_eof */
H5FD_poison_sec2_get_handle, /* get_handle */
H5FD_poison_sec2_read, /* read */
H5FD_poison_sec2_write, /* write */
H5FD_poison_sec2_read_vector, /* read_vector */
H5FD_poison_sec2_write_vector, /* write_vector */
H5FD_poison_sec2_read_selection, /* read_selection */
H5FD_poison_sec2_write_selection,/* write_selection */
H5FD_poison_sec2_flush, /* flush */
H5FD_poison_sec2_truncate, /* truncate */
H5FD_poison_sec2_lock, /* lock */
H5FD_poison_sec2_unlock, /* unlock */
H5FD_poison_sec2_delete, /* del */
H5FD_poison_sec2_ctl, /* ctl */
H5FD_FLMAP_DICHOTOMY /* fl_map */
};
hid_t H5FD_poison_sec2_init(void) {
if (H5FD_POISON_SEC2_id_g >= 0)
return H5FD_POISON_SEC2_id_g;
H5FD_POISON_SEC2_id_g = H5FDregister(&H5FD_poison_sec2_class_g);
return H5FD_POISON_SEC2_id_g;
}
herr_t H5Pset_fapl_poison_sec2(hid_t fapl_id) {
hid_t driver_id = H5FD_poison_sec2_init();
if (driver_id < 0)
return -1;
return H5Pset_driver(fapl_id, driver_id, NULL);
}
herr_t H5FD_poison_sec2_set_callback(
H5FD_poison_sec2_cb_t cb,
void *user_data
) {
H5FD_poison_sec2_cb_g = cb;
H5FD_poison_sec2_cb_ud_g = user_data;
return 0;
}
herr_t H5FD_poison_sec2_begin_fail_pass(void) {
H5FD_poison_sec2_clear_last_error();
H5FD_poison_sec2_fail_pass_g = true;
return 0;
}
herr_t H5FD_poison_sec2_end_fail_pass(void) {
H5FD_poison_sec2_fail_pass_g = false;
return 0;
}
int H5FD_poison_sec2_get_last_error(
const char **filename,
const char **operation,
int *error_number
) {
if (filename == NULL || operation == NULL || error_number == NULL)
return -1;
if (!H5FD_poison_sec2_last_error_g.valid) {
*filename = "";
*operation = "";
*error_number = 0;
return 0;
}
*filename = H5FD_poison_sec2_last_error_g.filename;
*operation = H5FD_poison_sec2_last_error_g.operation;
*error_number = H5FD_poison_sec2_last_error_g.error_number;
return 1;
}