Minor improvements to integrated NXmx workflow

Updates to CI pipeline

* New docker image for Ubuntu 22.04 with CMake 3.26
* New docker image for Rocky 9 with DIALS 3.27
* New automated test to check for DIALS processing with xia2.ssx

Reviewed-on: #42

.gitea/workflows/build_and_test.yml

@@ -76,27 +76,16 @@ jobs:
             upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/noble/nocuda/upload
     steps:
       - uses: actions/checkout@v4
-      - name: Install CMake 3.26 on Ubuntu 22.04
-        if: matrix.distro == 'ubuntu2204' || matrix.distro == 'ubuntu2204_nocuda'
-        shell: bash
-        run: |
-          set -euo pipefail
-          apt-get update
-          apt-get install -y wget gpg ca-certificates
-          wget -qO- https://apt.kitware.com/keys/kitware-archive-latest.asc \
-            | gpg --dearmor \
-            | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null
-          echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main" \
-            | tee /etc/apt/sources.list.d/kitware.list > /dev/null
-          apt-get update
-          apt-get install -y cmake=3.26.* cmake-data=3.26.* kitware-archive-keyring
-          cmake --version
-      - name: Build packages
+      - name: Setup build (cmake)
         shell: bash
         run: |
           mkdir -p build
           cd build
           cmake -G Ninja -DJFJOCH_INSTALL_DRIVER_SOURCE=ON -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake_flags }} ..
+      - name: Build packages
+        shell: bash
+        run: |
+          cd build
           ninja frontend
           ninja -j16 package
       - name: Upload packages
@@ -133,6 +122,32 @@ jobs:
                  python3 gitea_upload_file.py "$file"
             done
           fi
+  dials-test:
+    name: DIALS processing test
+    runs-on: jfjoch_rocky9
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build processing test
+        shell: bash
+        run: |
+          mkdir -p build
+          cd build
+          cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
+          ninja -j16 jfjoch_hdf5_test
+      - name: Generate test data (with virtual data set and 4 linked image files)
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p dials_test
+          cd dials_test
+          ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -f25 -V
+      - name: Run DIALS processing
+        shell: bash
+        run: |
+          source /opt/dials-v3-27-0/dials_env.sh
+          set -euo pipefail
+          cd dials_test
+          xia2.ssx image=writing_test_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000
   python-client:
     name: Generate python client
     runs-on: jfjoch_rocky8

CMakeLists.txt

@@ -175,7 +175,7 @@ ENDIF()
 
 IF (NOT JFJOCH_WRITER_ONLY)
     ADD_CUSTOM_COMMAND(OUTPUT frontend/dist/index.html
-            COMMAND npm install
+            COMMAND npm ci
             COMMAND npm run build
             COMMAND npm run redocly
             WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend)

broker/OpenAPIConvert.cpp

@@ -795,7 +795,7 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
     org::openapitools::server::model::File_writer_format ret;
     switch (input) {
         case FileWriterFormat::DataOnly:
-            ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NONE);
+            ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXONLYDATA);
             break;
         case FileWriterFormat::NXmxLegacy:
             ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXLEGACY);
@@ -803,6 +803,9 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
         case FileWriterFormat::NXmxVDS:
             ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXVDS);
             break;
+        case FileWriterFormat::NXmxIntegrated:
+            ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED);
+            break;
         case FileWriterFormat::CBF:
             ret.setValue(org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF);
             break;
@@ -820,12 +823,14 @@ org::openapitools::server::model::File_writer_format Convert(FileWriterFormat in
 
 FileWriterFormat Convert(const org::openapitools::server::model::File_writer_format& input) {
     switch (input.getValue()) {
-        case org::openapitools::server::model::File_writer_format::eFile_writer_format::NONE:
+        case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXONLYDATA:
             return FileWriterFormat::DataOnly;
         case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXLEGACY:
             return FileWriterFormat::NXmxLegacy;
         case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXVDS:
             return FileWriterFormat::NXmxVDS;
+        case org::openapitools::server::model::File_writer_format::eFile_writer_format::NXMXINTEGRATED:
+            return FileWriterFormat::NXmxIntegrated;
         case org::openapitools::server::model::File_writer_format::eFile_writer_format::CBF:
             return FileWriterFormat::CBF;
         case org::openapitools::server::model::File_writer_format::eFile_writer_format::TIFF:

broker/gen/model/File_writer_format.cpp

@@ -75,8 +75,8 @@ void to_json(nlohmann::json& j, const File_writer_format& o)
         case File_writer_format::eFile_writer_format::INVALID_VALUE_OPENAPI_GENERATED:
             j = "INVALID_VALUE_OPENAPI_GENERATED";
             break;
-        case File_writer_format::eFile_writer_format::NONE:
-            j = "None";
+        case File_writer_format::eFile_writer_format::NXMXONLYDATA:
+            j = "NXmxOnlyData";
             break;
         case File_writer_format::eFile_writer_format::NXMXLEGACY:
             j = "NXmxLegacy";
@@ -84,6 +84,9 @@ void to_json(nlohmann::json& j, const File_writer_format& o)
         case File_writer_format::eFile_writer_format::NXMXVDS:
             j = "NXmxVDS";
             break;
+        case File_writer_format::eFile_writer_format::NXMXINTEGRATED:
+            j = "NXmxIntegrated";
+            break;
         case File_writer_format::eFile_writer_format::CBF:
             j = "CBF";
             break;
@@ -100,8 +103,8 @@ void from_json(const nlohmann::json& j, File_writer_format& o)
 {
     
     auto s = j.get<std::string>();
-    if (s == "None") {
-     o.setValue(File_writer_format::eFile_writer_format::NONE);
+    if (s == "NXmxOnlyData") {
+     o.setValue(File_writer_format::eFile_writer_format::NXMXONLYDATA);
     } 
     else if (s == "NXmxLegacy") {
      o.setValue(File_writer_format::eFile_writer_format::NXMXLEGACY);
@@ -109,6 +112,9 @@ void from_json(const nlohmann::json& j, File_writer_format& o)
     else if (s == "NXmxVDS") {
      o.setValue(File_writer_format::eFile_writer_format::NXMXVDS);
     } 
+    else if (s == "NXmxIntegrated") {
+     o.setValue(File_writer_format::eFile_writer_format::NXMXINTEGRATED);
+    } 
     else if (s == "CBF") {
      o.setValue(File_writer_format::eFile_writer_format::CBF);
     } 

broker/gen/model/File_writer_format.h

@@ -12,7 +12,7 @@
 /*
  * File_writer_format.h
  *
- * None - no master file written NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia   NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling 
+ * NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia   NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset CBF - CBF format (limited metadata) TIFF - TIFF format (no metadata) 
  */
 
 #ifndef File_writer_format_H_
@@ -25,7 +25,7 @@ namespace org::openapitools::server::model
 {
 
 /// <summary>
-/// None - no master file written NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia   NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling 
+/// NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia   NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset CBF - CBF format (limited metadata) TIFF - TIFF format (no metadata) 
 /// </summary>
 class  File_writer_format
 {
@@ -38,9 +38,10 @@ public:
     // Avoiding name clashes with user defined
     // enum values
     INVALID_VALUE_OPENAPI_GENERATED = 0,
-    NONE, 
+    NXMXONLYDATA, 
     NXMXLEGACY, 
     NXMXVDS, 
+    NXMXINTEGRATED, 
     CBF, 
     TIFF, 
     NOFILEWRITTEN

broker/jfjoch_api.yaml

@@ -642,17 +642,22 @@ components:
     file_writer_format:
       type: string
       enum:
-        - "None"
+        - "NXmxOnlyData"
         - "NXmxLegacy"
         - "NXmxVDS"
+        - "NXmxIntegrated"
         - "CBF"
         - "TIFF"
         - "NoFileWritten"
       default: "NXmxLegacy"
       description: |
-        None - no master file written
+        NoFileWritten - no files are written at all
+        NXmxOnlyData - only data files are written, no master file
         NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia  
         NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
+        NXmxIntegrated - single HDF5 per dataset
+        CBF - CBF format (limited metadata)
+        TIFF - TIFF format (no metadata)
     file_writer_settings:
       type: object
       properties:

common/DiffractionExperiment.cpp

@@ -1083,7 +1083,8 @@ DiffractionExperiment &DiffractionExperiment::ImagesPerFile(int64_t input) {
 int64_t DiffractionExperiment::GetImagesPerFile() const {
     auto tmp = dataset.GetImagesPerFile();
 
-    if (tmp == 0)
+    if (tmp == 0
+        || file_writer.GetHDF5MasterFormatVersion() == FileWriterFormat::NXmxIntegrated)
         return GetImageNum();
     else
         return tmp;

common/JFJochMessages.h

@@ -33,7 +33,7 @@ enum class FileWriterFormat : int {
     DataOnly = 0,
     NXmxLegacy = 1,
     NXmxVDS = 2,
-    // TODO: NXmxTR = 3
+    NXmxIntegrated = 3,
     CBF = 4,
     TIFF = 5,
     NoFile = 6

docker/rocky9/Dockerfile

@@ -8,6 +8,9 @@ ARG EIGEN_TAG=3.4.0
 ARG LIBJPEG_TURBO_VERSION=3.1.2
 ARG LIBTIFF_VERSION=v4.7.1
 ARG HDF5_TAG="hdf5_1.14.6"
+ARG DIALS_VERSION=3.27.0
+ARG DIALS_TARBALL_URL=https://github.com/dials/dials/releases/download/v3.27.0/dials-v3-27-0-linux-x86_64.tar.xz
+ARG DIALS_PREFIX=/opt
 
 # Update base, enable CRB (RHEL/Rocky 9), and install toolchain + Qt deps
 RUN dnf -y update && \
@@ -65,6 +68,8 @@ RUN dnf -y update && \
       libdrm-devel \
       libglvnd-core-devel \
       libglvnd-devel \
+      glibc-langpack-en \
+      glibc-locale-source \
       freetype-devel && \
     dnf clean all && rm -rf /var/cache/dnf
 
@@ -185,9 +190,23 @@ RUN set -eux; \
     cmake --install .; \
     cd /; rm -rf /tmp/qt-everywhere-src-${QT_VERSION} /tmp/qt-everywhere-src-${QT_VERSION}.tar.xz
 
+# Install DIALS using the official binary tarball workflow
+RUN set -eux; \
+    cd /tmp; \
+    curl -fL -o dials.tar.xz "${DIALS_TARBALL_URL}"; \
+    tar -xJf dials.tar.xz; \
+    cd dials-installer; \
+    ./install --prefix="${DIALS_PREFIX}"; \
+    cd /; \
+    rm -rf /tmp/dials.tar.xz /tmp/dials-*-linux-x86_64
+
 # Make Qt and Eigen discoverable by CMake
 ENV CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/opt/hdf5-${HDF5_TAG}-static:/opt/qt-${QT_VERSION}-static
 
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
 # Set workdir for your project
 WORKDIR /workspace
 

docker/ubuntu2204/Dockerfile

@@ -22,11 +22,11 @@ RUN set -eux; \
       ca-certificates \
       curl \
       wget \
+      gpg \
       git \
       tar \
       xz-utils \
       build-essential \
-      cmake \
       ninja-build \
       python3 \
       python3-requests \
@@ -78,10 +78,25 @@ RUN set -eux; \
       libassimp-dev \
       libglvnd-dev \
       libfreetype6-dev; \
+    wget -qO- https://apt.kitware.com/keys/kitware-archive-latest.asc \
+      | gpg --dearmor \
+      | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null; \
+    echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main" \
+      | tee /etc/apt/sources.list.d/kitware.list > /dev/null; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+      cmake=3.26.* \
+      cmake-data=3.26.* \
+      kitware-archive-keyring; \
     apt-get -y install gcc-12 g++-12; \
     apt-get clean; \
     rm -rf /var/lib/apt/lists/*
 
+# Use GCC/G++ 12 for builds
+ENV CC=/usr/bin/gcc-12
+ENV CXX=/usr/bin/g++-12
+ENV PATH=/usr/bin:${PATH}
+
 # Build a static OpenSSL
 RUN set -eux; \
     cd /tmp; \
@@ -166,11 +181,6 @@ RUN set -eux; \
 ENV CMAKE_PREFIX_PATH=/opt/libtiff-${LIBTIFF_VERSION}-static:/opt/libjpeg-turbo-${LIBJPEG_TURBO_VERSION}-static
 ENV PKG_CONFIG_PATH=/opt/hdf5-${HDF5_TAG}-static/lib/pkgconfig:/opt/libjpeg-turbo-${LIBJPEG_TURBO_VERSION}-static/lib/pkgconfig:/opt/libtiff-${LIBTIFF_VERSION}-static/lib/pkgconfig:${OPENSSL_ROOT_DIR}/lib/pkgconfig:${OPENSSL_ROOT_DIR}/lib64/pkgconfig
 
-# Use GCC/G++ 12 for builds
-ENV CC=/usr/bin/gcc-12
-ENV CXX=/usr/bin/g++-12
-ENV PATH=/usr/bin:${PATH}
-
 # Build and install static Qt 6.9 with Core, Gui, Widgets, Charts, DBus
 ARG QT_PREFIX=/opt/qt-${QT_VERSION}-static
 RUN set -eux; \

docs/python_client/docs/FileWriterFormat.md

@@ -1,15 +1,17 @@
 # FileWriterFormat
 
-None - no master file written NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia   NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling 
+NoFileWritten - no files are written at all NXmxOnlyData - only data files are written, no master file NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia   NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling NXmxIntegrated - single HDF5 per dataset CBF - CBF format (limited metadata) TIFF - TIFF format (no metadata) 
 
 ## Enum
 
-* `NONE` (value: `'None'`)
+* `NXMXONLYDATA` (value: `'NXmxOnlyData'`)
 
 * `NXMXLEGACY` (value: `'NXmxLegacy'`)
 
 * `NXMXVDS` (value: `'NXmxVDS'`)
 
+* `NXMXINTEGRATED` (value: `'NXmxIntegrated'`)
+
 * `CBF` (value: `'CBF'`)
 
 * `TIFF` (value: `'TIFF'`)

frontend/src/components/FileWriterSettings.tsx

@@ -30,8 +30,8 @@ function stringToEnum(value: string): file_writer_format {
         (v) => v === value
     ) as file_writer_format;
 
-    // If no match is found, default to file_writer_format.NONE
-    return enumValue || file_writer_format.NONE;
+    // If no match is found, default to file_writer_format.NXMX_ONLY_DATA
+    return enumValue || file_writer_format.NXMX_ONLY_DATA;
 }
 
 
@@ -109,7 +109,8 @@ class FileWriterSettings extends React.Component<MyProps, MyState> {
                     >
                         <MenuItem value={file_writer_format.NXMX_LEGACY}>NXmx HDF5 master file with soft links (DECTRIS file writer compatibility)</MenuItem>
                         <MenuItem value={file_writer_format.NXMX_VDS}>NXmx HDF5 master file with virtual datasets</MenuItem>
-                        <MenuItem value={file_writer_format.NONE}>No NXmx HDF5 master file (only data files)</MenuItem>
+                        <MenuItem value={file_writer_format.NXMX_INTEGRATED}Single HDF5 file with data and metadata</MenuItem>
+                        <MenuItem value={file_writer_format.NXMX_ONLY_DATA}>No NXmx HDF5 master file (only data files)</MenuItem>
                         <MenuItem value={file_writer_format.CBF}>miniCBF (only data files; limited metadata)</MenuItem>
                         <MenuItem value={file_writer_format.TIFF}>TIFF (only data files; no metadata)</MenuItem>
                         <MenuItem value={file_writer_format.NO_FILE_WRITTEN}>No files saved</MenuItem>

frontend/src/openapi/models/file_writer_format.ts

@@ -4,15 +4,20 @@
 /* eslint-disable */
 
 /**
- * None - no master file written
+ * NoFileWritten - no files are written at all
+ * NXmxOnlyData - only data files are written, no master file
  * NXmxLegacy - legacy format with soft links to data files in the master file; necessary for DECTRIS Albula 4.0 and DECTRIS Neggia
  * NXmxVDS - newer format with virtual dataset linking data files in the master file, also includes better metadata handling
+ * NXmxIntegrated - single HDF5 per dataset
+ * CBF - CBF format (limited metadata)
+ * TIFF - TIFF format (no metadata)
  *
  */
 export enum file_writer_format {
-    NONE = 'None',
+    NXMX_ONLY_DATA = 'NXmxOnlyData',
     NXMX_LEGACY = 'NXmxLegacy',
     NXMX_VDS = 'NXmxVDS',
+    NXMX_INTEGRATED = 'NXmxIntegrated',
     CBF = 'CBF',
     TIFF = 'TIFF',
     NO_FILE_WRITTEN = 'NoFileWritten',

make_doc.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-python3.12 -m venv tmp_venv/
+set -euo pipefail
+
+python3.11 -m venv tmp_venv/
 source tmp_venv/bin/activate
 
 pip install -r docs/requirements.txt

tests/HDF5WritingTest.cpp

@@ -13,6 +13,64 @@
 
 using namespace std::literals::chrono_literals;
 
+TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
+    {
+        HDF5File file("scratch_group_reopen.h5");
+
+        REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
+        REQUIRE(file.Exists("/group1"));
+
+        REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
+        REQUIRE(file.Exists("/group1"));
+
+        REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
+    }
+
+    remove("scratch_group_reopen.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
+TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
+    const std::string first_value = "abc";
+    const std::string second_value = "a much longer attribute value";
+
+    {
+        HDF5File file("scratch_attr_string_update.h5");
+        REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
+        REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
+    }
+
+    {
+        HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
+        REQUIRE(file.ReadAttrStr("str_attr") == second_value);
+    }
+
+    remove("scratch_attr_string_update.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
+TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
+    const int64_t first_value = -1234567890123LL;
+    const int64_t second_value = 9876543210123LL;
+
+    {
+        HDF5File file("scratch_attr_int64_update.h5");
+        REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
+        REQUIRE(file.ReadAttrInt("int_attr") == first_value);
+
+        REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
+        REQUIRE(file.ReadAttrInt("int_attr") == second_value);
+    }
+
+    {
+        HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
+        REQUIRE(file.ReadAttrInt("int_attr") == second_value);
+    }
+
+    remove("scratch_attr_int64_update.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
 TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
     uint16_t tmp_scalar = 16788;
     {

tools/jfjoch_hdf5_test.cpp

@@ -18,26 +18,48 @@ int main(int argc, char **argv) {
 
     RegisterHDF5Filter();
 
-    if ((argc < 2) || (argc > 4)) {
-        std::cout << "Usage: ./jfjoch_hdf5_test <JF4M hdf5 file> {{<#images>} <rate in Hz>}" << std::endl;
-        std::cout << std::endl;
-        std::cout << "Env. variables:" << std::endl;
-        std::cout << "HDF5DATASET_WRITE_TEST_PREFIX" << std::endl;
-        std::cout << "HDF5MASTER_NEW_FORMAT" << std::endl;
+    int64_t nimages_out = 100;
+
+    std::string prefix = "writing_test";
+    FileWriterFormat format = FileWriterFormat::NXmxLegacy;
+    std::optional<int64_t> images_per_file;
+    std::optional<float> rotation;
+
+    int opt;
+    while ((opt = getopt(argc, argv, "o:n:Vf:R:")) != -1) {
+        switch (opt) {
+            case 'o':
+                prefix = optarg;
+                break;
+            case 'n':
+                nimages_out = atoll(optarg);
+                break;
+            case 'V':
+                format = FileWriterFormat::NXmxVDS;
+                break;
+            case 'R':
+                rotation = atof(optarg);
+                break;
+            case 'f':
+                images_per_file = atoll(optarg);
+                if (images_per_file.value() <= 0) {
+                    std::cerr << "Invalid number of images per file: " << optarg << std::endl;
+                    exit(EXIT_FAILURE);
+                }
+                images_per_file = atoll(optarg);
+                break;
+            default:
+                std::cout << "Usage: ./jfjoch_hdf5_test <JF4M hdf5 file> [-o <prefix>] [-n <num images>] [-V] [-f <num images per file>] [-R<rotation angle>]" << std::endl;
+                exit(EXIT_FAILURE);
+        }
+    }
+
+    if (optind >= argc) {
+        std::cout << "Usage: ./jfjoch_hdf5_test <JF4M hdf5 file> [-o <prefix>] [-n <num images>] [-V] [-f <num images per file>] [-R<rotation angle>]" << std::endl;
         exit(EXIT_FAILURE);
     }
 
-    int64_t nimages_out = 100;
-    double rate = 2200;
-
-    if (argc >= 3)
-        nimages_out = atoi(argv[2]);
-    if (argc >= 4)
-        rate = atof(argv[3]);
-
-    std::chrono::microseconds period_us((rate == 0) ? 0 : (int64_t) (1.0e6 / rate));
-
-    HDF5ReadOnlyFile data(argv[1]);
+    HDF5ReadOnlyFile data(argv[optind]);
     HDF5DataSet dataset(data, "/entry/data/data");
     HDF5DataSpace file_space(dataset);
 
@@ -53,6 +75,8 @@ int main(int argc, char **argv) {
     x.BeamX_pxl(1090).BeamY_pxl(1136).DetectorDistance_mm(75).IncidentEnergy_keV(WVL_1A_IN_KEV);
     x.MaskModuleEdges(true);
     x.MaskChipEdges(true);
+    if (rotation && rotation.value() != 0.0)
+        x.Goniometer(GoniometerAxis("omega", 0, rotation.value(), Coord(-1,0,0), std::nullopt));
 
     if ((file_space.GetDimensions()[1] == 2164) && (file_space.GetDimensions()[2] == 2068)) {
         std::cout << "JF4M with gaps detected (2068 x 2164)" << std::endl;
@@ -64,16 +88,12 @@ int main(int argc, char **argv) {
     logger.Info("Number of images in the original dataset: " + std::to_string(nimages));
 
     // Set file name
-    if (std::getenv("HDF5DATASET_WRITE_TEST_PREFIX") == nullptr)
-        x.FilePrefix("writing_test");
-    else
-        x.FilePrefix(std::getenv("HDF5DATASET_WRITE_TEST_PREFIX"));
+    x.FilePrefix(prefix);
+    x.SetFileWriterFormat(format);
+    x.OverwriteExistingFiles(true);
 
-    if (std::getenv("HDF5MASTER_NEW_FORMAT") != nullptr) {
-        std::cout << "Using new format for HDF5 master file" << std::endl;
-        x.SetFileWriterFormat(FileWriterFormat::NXmxVDS);
-    } else
-        x.SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
+    if (images_per_file.has_value())
+        x.ImagesPerFile(images_per_file.value());
 
     x.ImagesPerTrigger(nimages);
 
@@ -127,8 +147,6 @@ int main(int argc, char **argv) {
 
     size_t total_image_size = 0;
     for (int i = 0; i < nimages_out; i++) {
-        std::this_thread::sleep_until(start_time + i * period_us);
-
         DataMessage message{};
         message.image = CompressedImage(output[i % nimages], x.GetXPixelsNum(), x.GetYPixelsNum(),
                                         x.GetImageMode(), x.GetCompressionAlgorithm());

writer/HDF5NXmx.cpp

@@ -695,7 +695,7 @@ void NXmx::Finalize(const EndMessage &end) {
 
     if (start_message.file_format == FileWriterFormat::NXmxVDS)
         LinkToData_VDS(start_message, end);
-    else
+    else if (start_message.file_format == FileWriterFormat::NXmxLegacy)
         LinkToData(start_message, end);
 
     if (end.rotation_lattice)
@@ -731,3 +731,7 @@ void NXmx::UserData(const StartMessage &start) {
         }
     }
 }
+
+HDF5File *NXmx::GetFile() {
+    return hdf5_file.get();
+}

writer/HDF5NXmx.h

@@ -58,6 +58,8 @@ public:
     NXmx& operator=(const NXmx &other) = delete;
     void Finalize(const EndMessage &end);
     void WriteCalibration(const CompressedImage &image);
+
+    HDF5File *GetFile();
 };
 
 #endif //JUNGFRAUJOCH_HDF5NXMX_H

writer/HDF5Objects.cpp

@@ -315,76 +315,119 @@ void HDF5Fapl::SetVersionTo1p10orNewer() {
     H5Pset_libver_bounds(id, H5F_LIBVER_V110, H5F_LIBVER_LATEST);
 }
 
+template <typename T>
+static HDF5Object& WriteOrCreateScalarAttr(HDF5Object& object, const std::string& name, const T& val) {
+    HDF5DataSpace dataspace;
+    HDF5DataType datatype(val);
+
+    hid_t attr_id = -1;
+
+    if (H5Aexists(object.GetID(), name.c_str()) > 0) {
+        attr_id = H5Aopen(object.GetID(), name.c_str(), H5P_DEFAULT);
+        if (attr_id < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open attribute " + name);
+
+        hid_t existing_type = H5Aget_type(attr_id);
+        if (existing_type < 0) {
+            H5Aclose(attr_id);
+            throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot get attribute type " + name);
+        }
+
+        const bool recreate =
+                (H5Tget_class(existing_type) != H5Tget_class(datatype.GetID())) ||
+                (H5Tget_size(existing_type) != H5Tget_size(datatype.GetID()));
+
+        H5Tclose(existing_type);
+        H5Aclose(attr_id);
+
+        if (recreate) {
+            if (H5Adelete(object.GetID(), name.c_str()) < 0)
+                throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot delete attribute " + name);
+
+            attr_id = H5Acreate2(object.GetID(), name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
+        } else {
+            attr_id = H5Aopen(object.GetID(), name.c_str(), H5P_DEFAULT);
+        }
+    } else {
+        attr_id = H5Acreate2(object.GetID(), name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
+    }
+
+    if (attr_id < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot create/open attribute " + name);
+
+    herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
+    H5Aclose(attr_id);
+
+    if (ret < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5, "Attribute write unsuccessful");
+    return object;
+}
+
 HDF5Object & HDF5Object::Attr(const std::string &name, const std::string &val) {
     HDF5DataSpace dataspace;
     HDF5DataType datatype(val);
 
-    hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
+    hid_t attr_id = -1;
+
+    if (H5Aexists(id, name.c_str()) > 0) {
+        attr_id = H5Aopen(id, name.c_str(), H5P_DEFAULT);
+        if (attr_id < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open attribute " + name);
+
+        hid_t existing_type = H5Aget_type(attr_id);
+        if (existing_type < 0) {
+            H5Aclose(attr_id);
+            throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot get attribute type " + name);
+        }
+
+        const bool recreate =
+                (H5Tget_class(existing_type) != H5T_STRING) ||
+                (H5Tget_size(existing_type) < val.length() + 1);
+
+        H5Tclose(existing_type);
+        H5Aclose(attr_id);
+
+        if (recreate) {
+            if (H5Adelete(id, name.c_str()) < 0)
+                throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot delete attribute " + name);
+
+            attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
+        } else {
+            attr_id = H5Aopen(id, name.c_str(), H5P_DEFAULT);
+        }
+    } else {
+        attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
+    }
+
+    if (attr_id < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot create/open attribute " + name);
+
     herr_t ret = H5Awrite(attr_id, datatype.GetID(), val.c_str());
     H5Aclose(attr_id);
 
-    if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Attribute write unsuccessful");
+    if (ret < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5, "Attribute write unsuccessful");
     return *this;
 }
 
 HDF5Object & HDF5Object::Attr(const std::string &name, int32_t val) {
-    HDF5DataSpace dataspace;
-    HDF5DataType datatype(val);
-
-    hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
-    herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
-    H5Aclose(attr_id);
-
-    if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
-    return *this;
+    return WriteOrCreateScalarAttr(*this, name, val);
 }
 
 HDF5Object & HDF5Object::Attr(const std::string &name, uint32_t val) {
-    HDF5DataSpace dataspace;
-    HDF5DataType datatype(val);
-
-    hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
-    herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
-    H5Aclose(attr_id);
-
-    if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
-    return *this;
+    return WriteOrCreateScalarAttr(*this, name, val);
 }
 
 HDF5Object & HDF5Object::Attr(const std::string &name, int64_t val) {
-    HDF5DataSpace dataspace;
-    HDF5DataType datatype(val);
-
-    hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
-    herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
-    H5Aclose(attr_id);
-
-    if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
-    return *this;
+    return WriteOrCreateScalarAttr(*this, name, val);
 }
 
 HDF5Object & HDF5Object::Attr(const std::string &name, uint64_t val) {
-    HDF5DataSpace dataspace;
-    HDF5DataType datatype(val);
-
-    hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
-    herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
-    H5Aclose(attr_id);
-
-    if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
-    return *this;
+    return WriteOrCreateScalarAttr(*this, name, val);
 }
 
 HDF5Object & HDF5Object::Attr(const std::string &name, double val) {
-    HDF5DataSpace dataspace;
-    HDF5DataType datatype(val);
-
-    hid_t attr_id = H5Acreate2(id, name.c_str(), datatype.GetID(), dataspace.GetID(), H5P_DEFAULT, H5P_DEFAULT);
-    herr_t ret = H5Awrite(attr_id, datatype.GetID(), &val);
-    H5Aclose(attr_id);
-
-    if (ret < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Atrribute write unsucessful");
-    return *this;
+    return WriteOrCreateScalarAttr(*this, name, val);
 }
 
 HDF5Object & HDF5Object::Attr(const std::string &name, const std::vector<double> &val) {
@@ -632,14 +675,19 @@ HDF5Group::HDF5Group(const HDF5Object& parent, const std::string &name) : HDF5Gr
 }
 
 HDF5Group::HDF5Group(const HDF5Object& parent, const char *name) : HDF5Object() {
-    id = H5Gcreate(parent.GetID(), name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    if (H5Lexists(parent.GetID(), name, H5P_DEFAULT) > 0)
+        id = H5Gopen(parent.GetID(), name, H5P_DEFAULT);
+    else
+        id = H5Gcreate(parent.GetID(), name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+    if (id < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open/create HDF5 group " + std::string(name));
 }
 
 HDF5Group::~HDF5Group() {
     H5Gclose(id);
 }
 
-
 HDF5File::HDF5File(const std::string& filename, bool v1_10) : HDF5Object() {
     HDF5Fapl fapl;