v1.0.0-rc.166 (#76)

* `rugnux --mode calibration` writes `<prefix>.json` beside the `.poni`, whose `dataset_settings` member is a `jfjoch_broker` `dataset_settings` body as it stands.
* `rugnux` and `jfjoch_viewer` read PILATUS miniCBF sweeps natively, without conversion.
* Masters written by other facilities open, including Eiger 1.x and third-party NXmx variants.
* `rugnux` measures the beam centre on every run, and indexes with it when the file's value indexes nothing.
* A detector swung out on a 2theta arm is placed where the file says it stands, and the calibration can hold the tilt fixed.
* `rugnux` writes the unmerged MTZ by default, and a P1 merge beside it, so a wrong space group can be re-merged without reprocessing.
* Significant improvements to symmetry handling in `rugnux`: the lattice, the point group, the setting and the systematic absences.
* The `rugnux` report gives the resolution the CC1/2 fit reached, beside the range the reflections were written to.
* The `rugnux` report gives the twinning statistics measured before the space group was decided, beside the ones measured after.
* The `rugnux` report gives the strong-direction diffraction limit, and warns when CC1/2 is not monotone with resolution.
* `rugnux` ranks screw axes on the evidence their absences carry, rather than on how many control reflections a candidate happens to have.
* Twinning is no longer reported when the L-test contradicts it.
* The `rugnux` report gives the detector tilt, the measured tilt and the direct beam beside the beam centre, and a post-refined beam centre is judged against the run's own measurement rather than the file's.
* `--no-refine-tilt` holds the detector tilt at the value in the file, instead of zeroing it, when the calibration starts from the spots.
* The `jfjoch_viewer` grid scan view draws the cells in the proportion of the scan steps, so the map has the shape of the scanned area.

Reviewed-on: #76
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

writer/HDF5Objects.cpp

@@ -7,6 +7,11 @@
 
 #include <bitshuffle/bshuf_h5filter.h>
 #include "HDF5Objects.h"
+
+// HDF Group contributed LZ4 filter, the one DECTRIS Eiger firmware 1.x wrote its images with.
+// Read-only here: bshuf_h5filter.h defines 32008 for bitshuffle but nothing defines this one.
+#define LZ4_H5FILTER 32004
+
 #include "H5FDpoison_sec2.h"
 
 std::mutex hdf5_mutex;
@@ -996,9 +1001,20 @@ CompressionAlgorithm HDF5Dcpl::GetCompression() {
     size_t cd_nelemts = sizeof(cd_values);
     memset(cd_values, 0, sizeof(cd_values));
 
-    if (H5Pget_filter2(id, 0, nullptr, &cd_nelemts, cd_values, 0, nullptr, nullptr)
-        != BSHUF_H5FILTER)
-        throw JFJochException(JFJochExceptionCategory::HDF5,"Only Bitshuffle filter supported");
+    const auto filter = H5Pget_filter2(id, 0, nullptr, &cd_nelemts, cd_values, 0, nullptr, nullptr);
+
+    // Plain LZ4, no bit shuffle. DECTRIS Eiger firmware 1.x wrote images this way, and those files
+    // are still deposited; read-only, we never write it. Same block framing as bitshuffle - a 64-bit
+    // total size, a 32-bit block size, then 32-bit-prefixed blocks - which is why only the unshuffle
+    // step differs downstream.
+    if (filter == LZ4_H5FILTER)
+        return CompressionAlgorithm::LZ4_NO_SHUFFLE;
+
+    if (filter != BSHUF_H5FILTER)
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Unsupported HDF5 compression filter " + std::to_string(filter)
+                              + " (supported: bitshuffle " + std::to_string(BSHUF_H5FILTER)
+                              + ", LZ4 " + std::to_string(LZ4_H5FILTER) + ")");
 
     if (cd_values[4] == BSHUF_H5_COMPRESS_LZ4)
         return CompressionAlgorithm::BSHUF_LZ4;
@@ -1056,11 +1072,31 @@ std::string HDF5Object::GetString(const std::string &name, const std::string &de
 
 bool HDF5Object::Exists(const std::string &name) const {
     H5E_BEGIN_TRY {
-        return H5Lexists(GetID(), name.c_str(), H5P_DEFAULT) > 0;
+        // The LINK has to be there and the object it names has to be reachable. Those are two
+        // different questions for an external link, and every DECTRIS Eiger master asks the second
+        // one: it links saturation_value, pixel_mask, bit_depth_readout and serial_number into a
+        // companion <prefix>_meta.h5 that is routinely not kept when a dataset is archived or
+        // deposited. H5Lexists then says yes - the link is written in the master - and the read that
+        // follows throws, which turns every optional-field guard in the reader into a hard failure
+        // and refuses the whole file over a value it was prepared to do without.
+        if (H5Lexists(GetID(), name.c_str(), H5P_DEFAULT) <= 0)
+            return false;
+        return H5Oexists_by_name(GetID(), name.c_str(), H5P_DEFAULT) > 0;
     } H5E_END_TRY;
     return false;
 }
 
+bool HDF5Object::IsDataSet(const std::string &name) const {
+    hid_t dataset_id;
+    H5E_BEGIN_TRY {
+        dataset_id = H5Dopen(GetID(), name.c_str(), H5P_DEFAULT);
+    } H5E_END_TRY;
+    if (dataset_id < 0)
+        return false;
+    H5Dclose(dataset_id);
+    return true;
+}
+
 herr_t func(hid_t group, const char *name, const H5L_info2_t *info, void *op_data) {
     auto ret = reinterpret_cast<std::vector<std::string> *>(op_data);
     ret->emplace_back(name);
@@ -1090,7 +1126,7 @@ bool HDF5Object::IsExternalLink(const std::string& name) const {
     return (link_info.type == H5L_TYPE_EXTERNAL);
 }
 
-std::string HDF5Object::GetLinkedFileName(const std::string& name) const {
+std::pair<std::string, std::string> HDF5Object::GetLinkedTarget(const std::string& name) const {
     H5L_info2_t link_info;
 
     // Get information about the link
@@ -1122,9 +1158,11 @@ std::string HDF5Object::GetLinkedFileName(const std::string& name) const {
         throw JFJochException(JFJochExceptionCategory::HDF5,
                               "Failed to get link location");
 
-    std::string s(target_file_name);
+    return {target_file_name, target_object_path};
+}
 
-    return s;
+std::string HDF5Object::GetLinkedFileName(const std::string& name) const {
+    return GetLinkedTarget(name).first;
 }