diff --git a/reader/HDF5ImageLocator.cpp b/reader/HDF5ImageLocator.cpp index 4280276f0..3284d4970 100644 --- a/reader/HDF5ImageLocator.cpp +++ b/reader/HDF5ImageLocator.cpp @@ -63,8 +63,8 @@ HDF5ImageLocator::Location HDF5ImageLocator::Resolve(int64_t global_image) const if (layout_.format == FileWriterFormat::NXmxLegacy) { const uint32_t file_id = global_image / layout_.images_per_file; const uint32_t local_index = global_image % layout_.images_per_file; - const auto &path = layout_.legacy_files.at(file_id); - return {OpenCached(path), local_index, path}; + const auto &data_file = layout_.legacy_files.at(file_id); + return {OpenCached(data_file.path), local_index, data_file.path, data_file.dataset}; } if (layout_.format == FileWriterFormat::NXmxVDS @@ -161,7 +161,8 @@ std::vector HDF5ImageLocator::GetSourceMapping(uint64_t f throw JFJochException(JFJochExceptionCategory::HDF5, "Legacy image source file missing"); - AppendOrExtendSourceMapping(ret, layout_.legacy_files.at(file_id), "/entry/data/data", + const auto &data_file = layout_.legacy_files.at(file_id); + AppendOrExtendSourceMapping(ret, data_file.path, data_file.dataset, source_image, local_image, 1); } diff --git a/reader/HDF5ImageLocator.h b/reader/HDF5ImageLocator.h index 3d3f47f16..3e9585c5d 100644 --- a/reader/HDF5ImageLocator.h +++ b/reader/HDF5ImageLocator.h @@ -33,6 +33,13 @@ public: std::string dataset = "/entry/data/data"; }; + // One data file of a legacy multi-file dataset, with the dataset the master's link names + // inside it - the same "take the link at its word" the VDS branch already does. + struct LegacyFile { + std::string path; + std::string dataset; + }; + // Layout description, filled by the reader once the master file has been parsed. All paths // are absolute: legacy data files and VDS mapping filenames are resolved relative to the // master before being handed over, so the locator never deals with relative paths. @@ -41,7 +48,7 @@ public: HDF5DataSetLayout data_layout = HDF5DataSetLayout::CONTIGUOUS; std::shared_ptr master_file; std::string master_filename; - std::vector legacy_files; + std::vector legacy_files; size_t images_per_file = 1; std::vector vds_mappings; }; diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index f6c35c52d..47686e91b 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -104,7 +104,7 @@ inline std::pair parse_bravais_lattice(const std::st std::vector GetDimension(HDF5Object &object, const std::string &path) { const auto dim = object.GetDimension(path); if (dim.size() != 3) - throw JFJochException(JFJochExceptionCategory::HDF5, "Wrong dimension of /entry/data/data"); + throw JFJochException(JFJochExceptionCategory::HDF5, "Wrong dimension of " + path); return dim; } @@ -139,18 +139,42 @@ std::string ResolveRelativeToMaster(const std::string &directory, return (std::filesystem::path(directory) / path).string(); } -// DECTRIS Eiger firmware 1.x writes the same values under different names, and a file from that era -// is still what a repository hands you. The modern spelling is tried first; the legacy one is a pure -// fallback, and it is safe because every current Eiger master carries BOTH (measured on thirteen -// masters from ten facilities, firmware release-2020.2.1 through release-2024.1.1 - all of them -// write detector_distance and countrate_correction_count_cutoff beside the NXmx names). So this can -// never change what a modern file reads. -float ReadWithLegacyFallback(HDF5Object &file, const std::string &nxmx, const std::string &legacy) { - if (file.Exists(nxmx)) - return file.GetFloat(nxmx); - if (file.Exists(legacy)) - return file.GetFloat(legacy); - throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot find " + nxmx + " (nor " + legacy + ")"); +// A length in NXmx says which unit it is in, and a master written outside the DECTRIS toolchain +// uses that freedom: a Diamond-written one states pixel size, sensor thickness and detector +// distance in millimetres, which is correct NXmx. Reading those as metres is not a failure but a +// silent factor of a thousand, so the unit is read rather than assumed. An undeclared unit means +// metres - what every DECTRIS master and everything this system writes means by one. An unknown +// unit is refused rather than guessed at, for the same reason. +float ReadLength_m(HDF5Object &file, const std::string &name) { + HDF5DataSet dataset(file, name); + const float value = dataset.ReadScalar(); + + if (!dataset.AttrExists("units")) + return value; + + const std::string units = dataset.ReadAttrStr("units"); + if (units == "m") + return value; + if (units == "mm") + return value * 1e-3f; + if (units == "um") + return value * 1e-6f; + throw JFJochException(JFJochExceptionCategory::HDF5, name + ": unknown length unit " + units); +} + +// The same value under different names. DECTRIS Eiger firmware 1.x writes detector_distance where +// NXmx says distance, and a file from that era is still what a repository hands you; a Diamond +// master puts the distance one level up, in NXinstrument rather than in NXdetector. The NXmx +// spelling is tried first and the rest are pure fallbacks, which is safe because every current +// Eiger master carries BOTH DECTRIS names (measured on thirteen masters from ten facilities, +// firmware release-2020.2.1 through release-2024.1.1 - all of them write detector_distance and +// countrate_correction_count_cutoff beside the NXmx names). So this can never change what a +// modern file reads. ReadIntWithLegacyFallback below is the same arrangement for saturation_value. +float ReadLengthWithFallback_m(HDF5Object &file, const std::vector &names) { + for (const auto &name: names) + if (file.Exists(name)) + return ReadLength_m(file, name); + throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot find " + names.front()); } int64_t ReadIntWithLegacyFallback(HDF5Object &file, const std::string &nxmx, const std::string &legacy) { @@ -416,7 +440,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen // at the end. format stays NoFile if the master carries no image data. FileWriterFormat format = FileWriterFormat::NoFile; HDF5DataSetLayout data_layout = HDF5DataSetLayout::CONTIGUOUS; - std::vector legacy_format_files; + std::vector legacy_format_files; std::vector vds_data_mappings; size_t images_per_file = 1; @@ -511,11 +535,6 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen legacy_format_files.clear(); - image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector"); - image_size_y = master_file->GetInt("/entry/instrument/detector/detectorSpecific/y_pixels_in_detector"); - - //size_t expected_images = master_file->GetInt("/entry/instrument/detector/detectorSpecific/nimages"); - images_per_file = 0; number_of_images = 0; uint32_t nfiles = 0; @@ -531,14 +550,26 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen size_t fimages = 0; try { - auto fname = ResolveRelativeToMaster(directory.string(), - master_file->GetLinkedFileName(dname)); + // The link names both the file and the dataset inside it. DECTRIS and this + // system always call that dataset /entry/data/data, but a Diamond-written + // master links to a plain /data - so looking up a fixed name found nothing, + // and a full sweep read back as zero images. + const auto [linked_file, data_path] = master_file->GetLinkedTarget(dname); + const auto fname = ResolveRelativeToMaster(directory.string(), linked_file); HDF5ReadOnlyFile data_file(fname); - fimages = GetDimension(data_file, "/entry/data/data")[0]; + // The image size comes from the array itself, as it does for a VDS master. + // detectorSpecific/x_pixels_in_detector is a DECTRIS extension rather than + // NXmx, so a third-party writer has no reason to emit it. + const auto dim = GetDimension(data_file, data_path); + fimages = dim[0]; + if (nfiles == 0) { + image_size_y = dim[1]; + image_size_x = dim[2]; + } - legacy_format_files.push_back(fname); + legacy_format_files.push_back({fname, data_path}); if (nfiles == 0 && data_file.Exists("/entry/roi")) dataset->roi = data_file.FindLeafs("/entry/roi"); @@ -646,6 +677,10 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen number_of_images, fimages); } } catch (JFJochException &e) { + // Say why. Everything read here is optional per-image metadata except the + // image array itself, and losing that silently leaves an empty dataset that + // reads as "nothing to process" rather than as a broken file. + Logger("HDF5Reader").Warning("{}: {}", dname, e.what()); } if (nfiles == 0) @@ -653,6 +688,13 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen number_of_images += fimages; nfiles++; } + + // The master says where its images are. If none of them could be read, that is a + // broken dataset, not an empty one - processing it would end in a "no images" + // warning and a successful exit over a sweep that is sitting right there. + if (number_of_images == 0) + throw JFJochException(JFJochExceptionCategory::HDF5, + "Master file links to data files, but no images could be read from them"); } else { image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector"); image_size_y = master_file->GetInt("/entry/instrument/detector/detectorSpecific/y_pixels_in_detector"); @@ -684,9 +726,10 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen dataset->experiment.BeamX_pxl(master_file->GetFloat("/entry/instrument/detector/beam_center_x")); dataset->experiment.BeamY_pxl(master_file->GetFloat("/entry/instrument/detector/beam_center_y")); - float det_distance = ReadWithLegacyFallback(*master_file, - "/entry/instrument/detector/distance", - "/entry/instrument/detector/detector_distance"); + float det_distance = ReadLengthWithFallback_m(*master_file, + {"/entry/instrument/detector/distance", + "/entry/instrument/detector/detector_distance", + "/entry/instrument/detector_distance"}); if (det_distance < 0.001) det_distance = 0.1; // Set to 100 mm, if det distance is less than 1 mm dataset->experiment.DetectorDistance_mm(det_distance * 1000.0); @@ -849,7 +892,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen auto detector_name = master_file->GetString("/entry/instrument/detector/description"); DetectorSetup detector = DetDECTRIS(image_size_x, image_size_y, detector_name, {}); - detector.PixelSize_um(master_file->GetFloat("/entry/instrument/detector/x_pixel_size") * 1e6); + detector.PixelSize_um(ReadLength_m(*master_file, "/entry/instrument/detector/x_pixel_size") * 1e6); // Whether the stored image is mirrored in Y. A file written before this was recorded is // mirrored - that is the only thing Jungfraujoch has ever produced - so absence means true. detector.MirrorY(master_file @@ -870,9 +913,10 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen "/entry/instrument/detector/detectorSpecific/detector_orientation_quarter_turns") .value_or(0)))); // Sensor thickness/material drive the parallax/absorption model, so take them from the file - // rather than the DetectorSetup default (NXmx stores thickness in metres). + // rather than the DetectorSetup default. if (master_file->Exists("/entry/instrument/detector/sensor_thickness")) - detector.SensorThickness_um(master_file->GetFloat("/entry/instrument/detector/sensor_thickness") * 1e6); + detector.SensorThickness_um( + ReadLength_m(*master_file, "/entry/instrument/detector/sensor_thickness") * 1e6); if (master_file->Exists("/entry/instrument/detector/sensor_material")) detector.SensorMaterial(master_file->GetString("/entry/instrument/detector/sensor_material")); detector.SaturationLimit(SaturationLimitFromValue( @@ -916,17 +960,15 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen } if (image_size_x * image_size_y > 0) { - auto mask_tmp = master_file->ReadOptVector( - "/entry/instrument/detector/pixel_mask", - {0, 0}, - {image_size_y, image_size_x} - ); - if (mask_tmp.empty()) - mask_tmp = master_file->ReadOptVector( - "/entry/instrument/detector/detectorSpecific/pixel_mask", - {0, 0}, - {image_size_y, image_size_x} - ); + // IsDataSet, not Exists: a mask can be an external link into a file that does not hold + // it, which a deposition shipping no mask at all leaves behind. The link is there, so + // the name exists; only opening it says whether the array does. + std::vector mask_tmp; + for (const char *name: {"/entry/instrument/detector/pixel_mask", + "/entry/instrument/detector/detectorSpecific/pixel_mask"}) { + if (mask_tmp.empty() && master_file->IsDataSet(name)) + mask_tmp = master_file->ReadVector(name, {0, 0}, {image_size_y, image_size_x}); + } if (mask_tmp.empty()) mask_tmp = std::vector(image_size_x * image_size_y); dataset->pixel_mask = std::make_shared(mask_tmp); diff --git a/writer/HDF5Objects.cpp b/writer/HDF5Objects.cpp index ec9d5643b..9f951476f 100644 --- a/writer/HDF5Objects.cpp +++ b/writer/HDF5Objects.cpp @@ -1117,7 +1117,7 @@ bool HDF5Object::IsExternalLink(const std::string& name) const { return (link_info.type == H5L_TYPE_EXTERNAL); } -std::string HDF5Object::GetLinkedFileName(const std::string& name) const { +std::pair HDF5Object::GetLinkedTarget(const std::string& name) const { H5L_info2_t link_info; // Get information about the link @@ -1149,9 +1149,11 @@ std::string HDF5Object::GetLinkedFileName(const std::string& name) const { throw JFJochException(JFJochExceptionCategory::HDF5, "Failed to get link location"); - std::string s(target_file_name); + return {target_file_name, target_object_path}; +} - return s; +std::string HDF5Object::GetLinkedFileName(const std::string& name) const { + return GetLinkedTarget(name).first; } diff --git a/writer/HDF5Objects.h b/writer/HDF5Objects.h index d974a0ba5..9de50bdb0 100644 --- a/writer/HDF5Objects.h +++ b/writer/HDF5Objects.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "../common/JFJochException.h" #include "../compression/CompressionAlgorithmEnum.h" @@ -184,6 +185,10 @@ public: bool Exists(const std::string& name) const; bool IsDataSet(const std::string& name) const; bool IsExternalLink(const std::string& name) const; + // {file, object path} an external link points at. The object path matters: the master of a + // multi-file dataset names the dataset inside each data file, and a writer outside the DECTRIS + // toolchain is free to call it something other than /entry/data/data. + std::pair GetLinkedTarget(const std::string& name) const; std::string GetLinkedFileName(const std::string& name) const; std::vector FindLeafs(const std::string &name) const; std::vector GetDimension(const std::string &name);