Compare commits

..

3 Commits

Author SHA1 Message Date
6639b584ec Minor improvements to integrated NXmx workflow
Some checks failed
Build Packages / build:rpm (rocky8_nocuda) (push) Failing after 6m27s
Build Packages / build:rpm (rocky9_nocuda) (push) Failing after 6m48s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Failing after 8m21s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Failing after 9m18s
Build Packages / build:rpm (rocky8_sls9) (push) Failing after 9m20s
Build Packages / Generate python client (push) Successful in 49s
Build Packages / build:rpm (rocky9) (push) Failing after 10m21s
Build Packages / Create release (push) Has been skipped
Build Packages / build:rpm (rocky8) (push) Failing after 11m36s
Build Packages / Build documentation (push) Successful in 2m20s
Build Packages / build:rpm (rocky9_sls9) (push) Failing after 12m45s
Build Packages / build:rpm (ubuntu2404) (push) Failing after 7m55s
Build Packages / build:rpm (ubuntu2204) (push) Failing after 8m21s
Build Packages / DIALS processing test (push) Successful in 10m14s
Build Packages / Unit tests (push) Failing after 56m6s
2026-03-29 13:22:36 +02:00
cd0fa49f73 HDF5: Groups and attributes creation can be reused 2026-03-29 13:22:08 +02:00
91dd670043 OpenAPI: Add integrated NXmx file writer format 2026-03-28 20:07:25 +01:00
14 changed files with 218 additions and 76 deletions

View File

@@ -795,7 +795,7 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
org::openapitools::server::model::File_writer_format ret;
switch (input) {
case FileWriterFormat::DataOnly:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NONE);
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXONLYDATA);
break;
case FileWriterFormat::NXmxLegacy:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXLEGACY);
@@ -803,6 +803,9 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
case FileWriterFormat::NXmxVDS:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXVDS);
break;
case FileWriterFormat::NXmxIntegrated:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED);
break;
case FileWriterFormat::CBF:
ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF);
break;
@@ -820,12 +823,14 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
FileWriterFormat Convert(const org::openapitools::server::model::File_writer_format& input) {
switch (input.getValue()) {
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NONE:
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXONLYDATA:
return FileWriterFormat::DataOnly;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXLEGACY:
return FileWriterFormat::NXmxLegacy;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXVDS:
return FileWriterFormat::NXmxVDS;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED:
return FileWriterFormat::NXmxIntegrated;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF:
return FileWriterFormat::CBF;
case org::openapitools::server::model::File_writer_format::eFile_writer_format::TIFF:

View File

@@ -75,8 +75,8 @@ void to_json(nlohmann::json& j, const File_writer_format& o)
case File_writer_format::eFile_writer_format::INVALID_VALUE_OPENAPI_GENERATED:
j = "INVALID_VALUE_OPENAPI_GENERATED";
break;
case File_writer_format::eFile_writer_format::NONE:
j = "None";
case File_writer_format::eFile_writer_format::NXMXONLYDATA:
j = "NXmxOnlyData";
break;
case File_writer_format::eFile_writer_format::NXMXLEGACY:
j = "NXmxLegacy";
@@ -84,6 +84,9 @@ void to_json(nlohmann::json& j, const File_writer_format& o)
case File_writer_format::eFile_writer_format::NXMXVDS:
j = "NXmxVDS";
break;
case File_writer_format::eFile_writer_format::NXMXINTEGRATED:
j = "NXmxIntegrated";
break;
case File_writer_format::eFile_writer_format::CBF:
j = "CBF";
break;
@@ -100,8 +103,8 @@ void from_json(const nlohmann::json& j, File_writer_format& o)
{
auto s = j.get<std::string>();
if (s == "None") {
o.setValue(File_writer_format::eFile_writer_format::NONE);
if (s == "NXmxOnlyData") {
o.setValue(File_writer_format::eFile_writer_format::NXMXONLYDATA);
}
else if (s == "NXmxLegacy") {
o.setValue(File_writer_format::eFile_writer_format::NXMXLEGACY);
@@ -109,6 +112,9 @@ void from_json(const nlohmann::json& j, File_writer_format& o)
else if (s == "NXmxVDS") {
o.setValue(File_writer_format::eFile_writer_format::NXMXVDS);
}
else if (s == "NXmxIntegrated") {
o.setValue(File_writer_format::eFile_writer_format::NXMXINTEGRATED);
}
else if (s == "CBF") {
o.setValue(File_writer_format::eFile_writer_format::CBF);
}

View File

@@ -12,7 +12,7 @@
/*
* File_writer_format.h
*
* None - no master file written NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
* NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset CBF - CBF format (limited metadata) TIFF - TIFF format (no metadata)
*/
#ifndef File_writer_format_H_
@@ -25,7 +25,7 @@ namespace org::openapitools::server::model
{
/// <summary>
/// None - no master file written NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
/// NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset CBF - CBF format (limited metadata) TIFF - TIFF format (no metadata)
/// </summary>
class File_writer_format
{
@@ -38,9 +38,10 @@ public:
// Avoiding name clashes with user defined
// enum values
INVALID_VALUE_OPENAPI_GENERATED = 0,
NONE,
NXMXONLYDATA,
NXMXLEGACY,
NXMXVDS,
NXMXINTEGRATED,
CBF,
TIFF,
NOFILEWRITTEN

View File

@@ -642,17 +642,22 @@ components:
file_writer_format:
type: string
enum:
- "None"
- "NXmxOnlyData"
- "NXmxLegacy"
- "NXmxVDS"
- "NXmxIntegrated"
- "CBF"
- "TIFF"
- "NoFileWritten"
default: "NXmxLegacy"
description: |
None - no master file written
NoFileWritten - no files are written at all
NXmxOnlyData - only data files are written, no master file
NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia
NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
NXmxIntegrated - single HDF5 per dataset
CBF - CBF format (limited metadata)
TIFF - TIFF format (no metadata)
file_writer_settings:
type: object
properties:

File diff suppressed because one or more lines are too long

View File

@@ -1083,7 +1083,8 @@ DiffractionExperiment &DiffractionExperiment::ImagesPerFile(int64_t input) {
int64_t DiffractionExperiment::GetImagesPerFile() const {
auto tmp = dataset.GetImagesPerFile();
if (tmp == 0)
if (tmp == 0
|| file_writer.GetHDF5MasterFormatVersion() == FileWriterFormat::NXmxIntegrated)
return GetImageNum();
else
return tmp;

View File

@@ -33,7 +33,7 @@ enum class FileWriterFormat : int {
DataOnly = 0,
NXmxLegacy = 1,
NXmxVDS = 2,
// TODO: NXmxTR = 3
NXmxIntegrated = 3,
CBF = 4,
TIFF = 5,
NoFile = 6

View File

@@ -1,15 +1,17 @@
# FileWriterFormat
None - no master file written NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset CBF - CBF format (limited metadata) TIFF - TIFF format (no metadata)
## Enum
* `NONE` (value: `'None'`)
* `NXMXONLYDATA` (value: `'NXmxOnlyData'`)
* `NXMXLEGACY` (value: `'NXmxLegacy'`)
* `NXMXVDS` (value: `'NXmxVDS'`)
* `NXMXINTEGRATED` (value: `'NXmxIntegrated'`)
* `CBF` (value: `'CBF'`)
* `TIFF` (value: `'TIFF'`)

View File

@@ -30,8 +30,8 @@ function stringToEnum(value: string): file_writer_format {
(v) => v === value
) as file_writer_format;
// If no match is found, default to file_writer_format.NONE
return enumValue || file_writer_format.NONE;
// If no match is found, default to file_writer_format.NXMX_ONLY_DATA
return enumValue || file_writer_format.NXMX_ONLY_DATA;
}
@@ -109,7 +109,8 @@ class FileWriterSettings extends React.Component<MyProps, MyState> {
>
<MenuItem value={file_writer_format.NXMX_LEGACY}>NXmx HDF5 master file with soft links (DECTRIS file writer compatibility)</MenuItem>
<MenuItem value={file_writer_format.NXMX_VDS}>NXmx HDF5 master file with virtual datasets</MenuItem>
<MenuItem value={file_writer_format.NONE}>No NXmx HDF5 master file (only data files)</MenuItem>
<MenuItem value={file_writer_format.NXMX_INTEGRATED}Single HDF5 file with data and metadata</MenuItem>
<MenuItem value={file_writer_format.NXMX_ONLY_DATA}>No NXmx HDF5 master file (only data files)</MenuItem>
<MenuItem value={file_writer_format.CBF}>miniCBF (only data files; limited metadata)</MenuItem>
<MenuItem value={file_writer_format.TIFF}>TIFF (only data files; no metadata)</MenuItem>
<MenuItem value={file_writer_format.NO_FILE_WRITTEN}>No files saved</MenuItem>

View File

@@ -4,15 +4,20 @@
/* eslint-disable */
/**
* None - no master file written
* NoFileWritten - no files are written at all
* NXmxOnlyData - only data files are written, no master file
* NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia
* NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
* NXmxIntegrated - single HDF5 per dataset
* CBF - CBF format (limited metadata)
* TIFF - TIFF format (no metadata)
*
*/
export enum file_writer_format {
NONE = 'None',
NXMX_ONLY_DATA = 'NXmxOnlyData',
NXMX_LEGACY = 'NXmxLegacy',
NXMX_VDS = 'NXmxVDS',
NXMX_INTEGRATED = 'NXmxIntegrated',
CBF = 'CBF',
TIFF = 'TIFF',
NO_FILE_WRITTEN = 'NoFileWritten',

View File

@@ -13,6 +13,64 @@
using namespace std::literals::chrono_literals;
TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
{
HDF5File file("scratch_group_reopen.h5");
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
REQUIRE(file.Exists("/group1"));
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
REQUIRE(file.Exists("/group1"));
REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
}
remove("scratch_group_reopen.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
const std::string first_value = "abc";
const std::string second_value = "a much longer attribute value";
{
HDF5File file("scratch_attr_string_update.h5");
REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
}
{
HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
REQUIRE(file.ReadAttrStr("str_attr") == second_value);
}
remove("scratch_attr_string_update.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
const int64_t first_value = -1234567890123LL;
const int64_t second_value = 9876543210123LL;
{
HDF5File file("scratch_attr_int64_update.h5");
REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
REQUIRE(file.ReadAttrInt("int_attr") == first_value);
REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
}
{
HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
}
remove("scratch_attr_int64_update.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{

View File

@@ -695,7 +695,7 @@ void NXmx::Finalize(const EndMessage &end) {
if (start_message.file_format == FileWriterFormat::NXmxVDS)
LinkToData_VDS(start_message, end);
else
else if (start_message.file_format == FileWriterFormat::NXmxLegacy)
LinkToData(start_message, end);
if (end.rotation_lattice)
@@ -731,3 +731,7 @@ void NXmx::UserData(const StartMessage &start) {
}
}
}
HDF5File *NXmx::GetFile() {
return hdf5_file.get();
}

View File

@@ -58,6 +58,8 @@ public:
NXmx& operator=(const NXmx &other) = delete;
void Finalize(const EndMessage &end);
void WriteCalibration(const CompressedImage &image);
HDF5File *GetFile();
};
#endif //JUNGFRAUJOCH_HDF5NXMX_H

View File

@@ -315,76 +315,119 @@ void HDF5Fapl::SetVersionTo1p10orNewer() {
H5Pset_libver_bounds(id, H5F_LIBVER_V110, H5F_LIBVER_LATEST);
}
template <typename T>
static HDF5Object& WriteOrCreateScalarAttr(HDF5Object& object, const std::string& name, const T& val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = -1;
if (H5Aexists(object.GetID(), name.c_str()) > 0) {
attr_id = H5Aopen(object.GetID(), name.c_str(), H5P_DEFAULT);
if (attr_id < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open attribute " + name);
hid_t existing_type = H5Aget_type(attr_id);
if (existing_type < 0) {
H5Aclose(attr_id);
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot get attribute type " + name);
}
const bool recreate =
(H5Tget_class(existing_type) != H5Tget_class(datatype.GetID())) ||
(H5Tget_size(existing_type) != H5Tget_size(datatype.GetID()));
H5Tclose(existing_type);
H5Aclose(attr_id);
if (recreate) {
if (H5Adelete(object.GetID(), name.c_str()) < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot delete attribute " + name);
attr_id = H5Acreate2(object.GetID(), name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
} else {
attr_id = H5Aopen(object.GetID(), name.c_str(), H5P_DEFAULT);
}
} else {
attr_id = H5Acreate2(object.GetID(), name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
}
if (attr_id < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot create/open attribute " + name);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
H5Aclose(attr_id);
if (ret < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Attribute write unsuccessful");
return object;
}
HDF5Object & HDF5Object::Attr(const std::string &name, const std::string &val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
hid_t attr_id = -1;
if (H5Aexists(id, name.c_str()) > 0) {
attr_id = H5Aopen(id, name.c_str(), H5P_DEFAULT);
if (attr_id < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open attribute " + name);
hid_t existing_type = H5Aget_type(attr_id);
if (existing_type < 0) {
H5Aclose(attr_id);
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot get attribute type " + name);
}
const bool recreate =
(H5Tget_class(existing_type) != H5T_STRING) ||
(H5Tget_size(existing_type) < val.length() + 1);
H5Tclose(existing_type);
H5Aclose(attr_id);
if (recreate) {
if (H5Adelete(id, name.c_str()) < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot delete attribute " + name);
attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
} else {
attr_id = H5Aopen(id, name.c_str(), H5P_DEFAULT);
}
} else {
attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
}
if (attr_id < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot create/open attribute " + name);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), val.c_str());
H5Aclose(attr_id);
if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Attribute write unsuccessful");
if (ret < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Attribute write unsuccessful");
return *this;
}
HDF5Object & HDF5Object::Attr(const std::string &name, int32_t val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
H5Aclose(attr_id);
if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
return *this;
return WriteOrCreateScalarAttr(*this, name, val);
}
HDF5Object & HDF5Object::Attr(const std::string &name, uint32_t val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
H5Aclose(attr_id);
if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
return *this;
return WriteOrCreateScalarAttr(*this, name, val);
}
HDF5Object & HDF5Object::Attr(const std::string &name, int64_t val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
H5Aclose(attr_id);
if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
return *this;
return WriteOrCreateScalarAttr(*this, name, val);
}
HDF5Object & HDF5Object::Attr(const std::string &name, uint64_t val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
H5Aclose(attr_id);
if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
return *this;
return WriteOrCreateScalarAttr(*this, name, val);
}
HDF5Object & HDF5Object::Attr(const std::string &name, double val) {
HDF5DataSpace dataspace;
HDF5DataType datatype(val);
hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
H5Aclose(attr_id);
if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
return *this;
return WriteOrCreateScalarAttr(*this, name, val);
}
HDF5Object & HDF5Object::Attr(const std::string &name, const std::vector<double> &val) {
@@ -632,14 +675,19 @@ HDF5Group::HDF5Group(const HDF5Object& parent, const std::string &name) : HDF5Gr
}
HDF5Group::HDF5Group(const HDF5Object& parent, const char *name) : HDF5Object() {
id = H5Gcreate(parent.GetID(), name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (H5Lexists(parent.GetID(), name, H5P_DEFAULT) > 0)
id = H5Gopen(parent.GetID(), name, H5P_DEFAULT);
else
id = H5Gcreate(parent.GetID(), name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (id < 0)
throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open/create HDF5 group " + std::string(name));
}
HDF5Group::~HDF5Group() {
H5Gclose(id);
}
HDF5File::HDF5File(const std::string& filename, bool v1_10) : HDF5Object() {
HDF5Fapl fapl;