From 809441d0f06a75c6e41765a7353e74c293d45ed5 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 11 Apr 2024 16:10:21 +0200 Subject: [PATCH] Move back to DECTRIS flavor of NXmx --- .gitlab-ci.yml | 26 +- frontend_ui/src/components/PreviewImage.tsx | 2 +- tests/HDF5WritingTest.cpp | 254 ++++++++------------ tests/JFJochReceiverProcessingTest.cpp | 2 +- tests/StreamWriterTest.cpp | 4 +- tests/crystfel/jf4m.geom | 15 +- tests/crystfel/writing_test.lst | 2 +- tests/xds_durin/XDS.INP | 15 +- tests/xds_neggia/XDS.INP | 73 ++++++ writer/HDF5DataFile.cpp | 9 +- writer/HDF5DataFile.h | 5 +- writer/HDF5NXmx.cpp | 56 +---- writer/HDF5Objects.cpp | 4 +- writer/HDF5Sum.cpp | 2 +- writer/HDF5Writer.cpp | 4 +- 15 files changed, 241 insertions(+), 232 deletions(-) create mode 100644 tests/xds_neggia/XDS.INP diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1269e6fb..0746516a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,7 +167,31 @@ test:x86:xds_durin: - cmake -DCMAKE_BUILD_TYPE=Release .. - make -j8 HDF5DatasetWriteTest - cd ../tests/xds_durin - - HDF5DATASET_WRITE_TEST_IMAGES_PER_FILE=0 ../../build/tools/HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 100 + - HDF5DATASET_WRITE_TEST_IMAGES_PER_FILE=0 ../../build/tools/HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 25 + - xds_par |grep -a1 ISa |tail -n1 + +test:x86:xds_neggia: + stage: test + timeout: 90m + variables: + GIT_SUBMODULE_STRATEGY: recursive + CTEST_OUTPUT_ON_FAILURE: 1 + CC: gcc + CXX: g++ + needs: ["build:x86:gcc"] + dependencies: [] + tags: + - gcc + - x86 + - xds + script: + - source /opt/rh/gcc-toolset-12/enable + - mkdir -p build + - cd build + - cmake -DCMAKE_BUILD_TYPE=Release .. + - make -j8 HDF5DatasetWriteTest + - cd ../tests/xds_neggia + - HDF5DATASET_WRITE_TEST_IMAGES_PER_FILE=0 ../../build/tools/HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 25 - xds_par |grep -a1 ISa |tail -n1 test:x86:xia2.ssx: diff --git a/frontend_ui/src/components/PreviewImage.tsx b/frontend_ui/src/components/PreviewImage.tsx index 11d3bfbc..a416a048 100644 --- a/frontend_ui/src/components/PreviewImage.tsx +++ b/frontend_ui/src/components/PreviewImage.tsx @@ -129,7 +129,7 @@ class PreviewImage extends Component { Show only indexed images   
Saturation value     diff --git a/tests/HDF5WritingTest.cpp b/tests/HDF5WritingTest.cpp index f9a30b28..ecfbb510 100644 --- a/tests/HDF5WritingTest.cpp +++ b/tests/HDF5WritingTest.cpp @@ -274,155 +274,9 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") { REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("HDF5Writer_VDS", "[HDF5][Full]") { - DiffractionExperiment x(DetectorGeometry(1)); - - x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds"); - - { - RegisterHDF5Filter(); - - StartMessage start_message; - x.FillMessage(start_message); - - EndMessage end_message; - end_message.max_image_number = x.GetImageNum(); - - HDF5Writer writer(start_message); - std::vector image(x.GetPixelsNum()); - std::vector spots; - for (int i = 0; i < x.GetImageNum(); i++) { - for (auto &j: image) - j = i; - DataMessage message{}; - message.image.pixel_depth_bytes = 2; - message.image.pixel_is_signed = false; - message.image.xpixel = x.GetXPixelsNum(); - message.image.ypixel = x.GetYPixelsNum(); - message.image.algorithm = CompressionAlgorithm::NO_COMPRESSION; - message.image.data = (uint8_t *) image.data(); - message.image.size = x.GetPixelsNum() * x.GetPixelDepth(); - message.spots = spots; - message.number = i; - - REQUIRE_NOTHROW(writer.Write(message)); - } - - REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message)); - } - { - HDF5ReadOnlyFile file("vds_master.h5"); - std::unique_ptr dataset; - REQUIRE_NOTHROW(dataset = std::make_unique(file,"/entry/data/data")); - HDF5DataSpace file_space(*dataset); - REQUIRE(file_space.GetNumOfDimensions() == 3); - REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum()); - REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum()); - REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum()); - - std::vector v(x.GetImageNum() * x.GetYPixelsNum() * x.GetXPixelsNum()); - std::vector start = {0, 0, 0}; - dataset->ReadVector(v, start, file_space.GetDimensions()); - - REQUIRE(v[0] == 0); - REQUIRE(v[x.GetYPixelsNum() * x.GetXPixelsNum()] == 1); - REQUIRE(v[2 * x.GetYPixelsNum() * x.GetXPixelsNum()] == 2); - REQUIRE(v[3 * x.GetYPixelsNum() * x.GetXPixelsNum()] == 3); - REQUIRE(v[4 * x.GetYPixelsNum() * x.GetXPixelsNum()] == 4); - } - // No leftover HDF5 objects - REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); -} - -TEST_CASE("HDF5Writer_VDS_missing", "[HDF5][Full]") { - DiffractionExperiment x(DetectorGeometry(1)); - x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_missing"); - - { - RegisterHDF5Filter(); - - StartMessage start_message; - x.FillMessage(start_message); - - EndMessage end_message; - end_message.max_image_number = x.GetImageNum() - 1; - - HDF5Writer writer(start_message); - - std::vector image(x.GetPixelsNum()); - std::vector spots; - - for (int i = 0; i < x.GetImageNum() - 1; i++) { - for (auto &j: image) - j = i; - DataMessage message{}; - message.image.pixel_depth_bytes = 2; - message.image.pixel_is_signed = false; - message.image.xpixel = x.GetXPixelsNum(); - message.image.ypixel = x.GetYPixelsNum(); - message.image.algorithm = CompressionAlgorithm::NO_COMPRESSION; - message.image.data = (uint8_t *) image.data(); - message.image.size = x.GetPixelsNum() * x.GetPixelDepth(); - message.spots = spots; - message.number = i; - - REQUIRE_NOTHROW(writer.Write(message)); - } - - REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message)); - } - { - HDF5ReadOnlyFile file("vds_missing_master.h5"); - std::unique_ptr dataset; - REQUIRE_NOTHROW(dataset = std::make_unique(file,"/entry/data/data")); - HDF5DataSpace file_space(*dataset); - REQUIRE(file_space.GetNumOfDimensions() == 3); - REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum() -1); - REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum()); - REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum()); - - std::vector v((x.GetImageNum() - 1) * x.GetYPixelsNum() * x.GetXPixelsNum()); - std::vector start = {0, 0, 0}; - dataset->ReadVector(v, start, file_space.GetDimensions()); - - REQUIRE(v[0] == 0); - REQUIRE(v[x.GetYPixelsNum() * x.GetXPixelsNum()] == 1); - REQUIRE(v[2 * x.GetYPixelsNum() * x.GetXPixelsNum()] == 2); - REQUIRE(v[3 * x.GetYPixelsNum() * x.GetXPixelsNum()] == 3); - } - // No leftover HDF5 objects - REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); -} - - -TEST_CASE("HDF5Writer_VDS_zero_images", "[HDF5][Full]") { - DiffractionExperiment x(DetectorGeometry(1)); - x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("vds_zero"); - - { - RegisterHDF5Filter(); - - StartMessage start_message; - x.FillMessage(start_message); - - EndMessage end_message; - end_message.max_image_number = 0; - - REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message)); - } - { - std::unique_ptr file; - std::unique_ptr dataset; - REQUIRE_NOTHROW(file = std::make_unique("vds_zero_master.h5")); - REQUIRE_THROWS(dataset = std::make_unique(*file,"/entry/data/data")); - } - // No leftover HDF5 objects - REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); -} - TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") { - REQUIRE(HDF5Metadata::DataFileName("z/x", 34) == "z/x_data_000034.h5"); - REQUIRE(HDF5Metadata::DataFileName("z/x", 0) == "z/x_data_000000.h5"); + REQUIRE(HDF5Metadata::DataFileName("z/x", 34) == "z/x_data_000035.h5"); + REQUIRE(HDF5Metadata::DataFileName("z/x", 0) == "z/x_data_000001.h5"); REQUIRE_THROWS(HDF5Metadata::DataFileName("z/x", 1000000)); REQUIRE_THROWS(HDF5Metadata::DataFileName("z/x", -1)); } @@ -447,4 +301,106 @@ TEST_CASE("HDF5DataType", "[HDF5]") { REQUIRE(type8.GetElemSize() == 8); REQUIRE_THROWS(HDF5DataType(7,true)); -} \ No newline at end of file +} + +TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") { + DiffractionExperiment x(DetectorGeometry(1)); + + x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link"); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + + EndMessage end_message; + end_message.max_image_number = x.GetImageNum() - 2; + + HDF5Writer writer(start_message); + std::vector image(x.GetPixelsNum()); + std::vector spots; + for (int i = 0; i < x.GetImageNum() - 2; i++) { + for (auto &j: image) + j = i; + DataMessage message{}; + message.image.pixel_depth_bytes = 2; + message.image.pixel_is_signed = false; + message.image.xpixel = x.GetXPixelsNum(); + message.image.ypixel = x.GetYPixelsNum(); + message.image.algorithm = CompressionAlgorithm::NO_COMPRESSION; + message.image.data = (uint8_t *) image.data(); + message.image.size = x.GetPixelsNum() * x.GetPixelDepth(); + message.spots = spots; + message.number = i; + + REQUIRE_NOTHROW(writer.Write(message)); + } + + REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message)); + } + { + HDF5ReadOnlyFile file("link_master.h5"); + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file,"/entry/data/data_000001")); + HDF5DataSpace file_space(*dataset); + REQUIRE(file_space.GetNumOfDimensions() == 3); + REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile()); + REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum()); + REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum()); + } + { + HDF5ReadOnlyFile file("link_master.h5"); + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file,"/entry/data/data_000002")); + HDF5DataSpace file_space(*dataset); + REQUIRE(file_space.GetNumOfDimensions() == 3); + REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile()); + REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum()); + REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum()); + } + { + HDF5ReadOnlyFile file("link_master.h5"); + std::unique_ptr dataset; + REQUIRE_NOTHROW(dataset = std::make_unique(file,"/entry/data/data_000003")); + HDF5DataSpace file_space(*dataset); + REQUIRE(file_space.GetNumOfDimensions() == 3); + REQUIRE(file_space.GetDimensions()[0] == 1); + REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum()); + REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum()); + } + { + HDF5ReadOnlyFile file("link_master.h5"); + std::unique_ptr dataset; + REQUIRE_THROWS(dataset = std::make_unique(file,"/entry/data/data_000004")); + } + + // No leftover HDF5 objects + REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} + + +TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") { + DiffractionExperiment x(DetectorGeometry(1)); + x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero"); + + { + RegisterHDF5Filter(); + + StartMessage start_message; + x.FillMessage(start_message); + + EndMessage end_message; + end_message.max_image_number = 0; + + REQUIRE_NOTHROW(HDF5Metadata::NXmx(start_message, end_message)); + } + { + std::unique_ptr file; + std::unique_ptr dataset; + REQUIRE_NOTHROW(file = std::make_unique("link_zero_master.h5")); + REQUIRE_THROWS(dataset = std::make_unique(*file,"/entry/data/data_000001")); + } + // No leftover HDF5 objects + REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} diff --git a/tests/JFJochReceiverProcessingTest.cpp b/tests/JFJochReceiverProcessingTest.cpp index dd648c6b..378dfd39 100644 --- a/tests/JFJochReceiverProcessingTest.cpp +++ b/tests/JFJochReceiverProcessingTest.cpp @@ -19,7 +19,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index", "[JFJochReceiver] const uint16_t nthreads = 4; DiffractionExperiment experiment(DetectorGeometry(8,2,8,36)); - experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true) + experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2) .FilePrefix("lyso_test").ConversionOnFPGA(false) .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4) .SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90}); diff --git a/tests/StreamWriterTest.cpp b/tests/StreamWriterTest.cpp index 6ad1f520..a7a139e3 100644 --- a/tests/StreamWriterTest.cpp +++ b/tests/StreamWriterTest.cpp @@ -46,7 +46,7 @@ TEST_CASE("StreamWriterTest_ZMQ","[JFJochWriter]") { // HDF5 file can be opened std::unique_ptr file; - REQUIRE_NOTHROW(file = std::make_unique("subdir/JFJochWriterTest_data_000000.h5")); + REQUIRE_NOTHROW(file = std::make_unique("subdir/JFJochWriterTest_data_000001.h5")); std::unique_ptr dataset; REQUIRE_NOTHROW(dataset = std::make_unique(*file, "/entry/data/data")); std::unique_ptr dataspace; @@ -58,6 +58,6 @@ TEST_CASE("StreamWriterTest_ZMQ","[JFJochWriter]") { REQUIRE(dataspace->GetDimensions()[2] == 2*RAW_MODULE_LINES); REQUIRE(std::filesystem::remove("subdir/JFJochWriterTest_master.h5")); - REQUIRE(std::filesystem::remove("subdir/JFJochWriterTest_data_000000.h5")); + REQUIRE(std::filesystem::remove("subdir/JFJochWriterTest_data_000001.h5")); REQUIRE(std::filesystem::remove("subdir")); } diff --git a/tests/crystfel/jf4m.geom b/tests/crystfel/jf4m.geom index 386465b5..67ac7f2d 100644 --- a/tests/crystfel/jf4m.geom +++ b/tests/crystfel/jf4m.geom @@ -40,12 +40,6 @@ bad_h2/min_ss = 1614 bad_h2/max_fs = 2067 bad_h2/max_ss = 1649 -; Uncomment these lines if you have a separate bad pixel map (recommended!) -;mask_file = eiger-badmap.h5 -mask = /entry/instrument/detector/pixel_mask -mask_good = 0x0 -mask_bad = 0xFF - ; corner_{x,y} set the position of the corner of the detector (in pixels) ; relative to the beam 0/min_fs = 0 @@ -54,10 +48,5 @@ mask_bad = 0xFF 0/max_ss = 2163 0/corner_x = -1090.72 0/corner_y = -1137.24 -0/fs = +1.000000x +0.000373y -0/ss = -0.000373x +1.000000y - - - - -0/coffset = -0.000001 +0/fs = +1.000000x +0.000000y +0/ss = -0.000000x +1.000000y diff --git a/tests/crystfel/writing_test.lst b/tests/crystfel/writing_test.lst index 52acb432..e71d33ef 100644 --- a/tests/crystfel/writing_test.lst +++ b/tests/crystfel/writing_test.lst @@ -1 +1 @@ -writing_test_master.h5 +writing_test_data_000001.h5 diff --git a/tests/xds_durin/XDS.INP b/tests/xds_durin/XDS.INP index ac6c9181..0751c9b7 100644 --- a/tests/xds_durin/XDS.INP +++ b/tests/xds_durin/XDS.INP @@ -2,16 +2,9 @@ !Settings for EIGER16M @ X06SA SLS !job control JOB=XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT -!JOB=IDXREF -!JOB=DEFPIX INTEGRATE CORRECT -!JOB=CORRECT -MAXIMUM_NUMBER_OF_JOBS=4 -MAXIMUM_NUMBER_OF_PROCESSORS=12 -!MINUTE=0 !how long to wait (minutes), inplies only one JOBS -!CLUSTER_NODES=x06sa-cn-111 x06sa-cn-112 x06sa-cn-113 x06sa-cn-114 x06sa-cn-115 x06sa-cn-116 x06sa-cn-117 x06sa-cn-118 -!SPACE_GROUP_NUMBER= 89 -!UNIT_CELL_CONSTANTS= 31.5 31.5 97.8 90.000 90.000 90.000 +MAXIMUM_NUMBER_OF_JOBS=1 +MAXIMUM_NUMBER_OF_PROCESSORS=1 ! for this experiment: ORGX= 1097 @@ -20,9 +13,9 @@ DETECTOR_DISTANCE= 75 OSCILLATION_RANGE= 0.088 X-RAY_WAVELENGTH= 1.0 NAME_TEMPLATE_OF_DATA_FRAMES=writing_test_??????.h5 -DATA_RANGE= 1 100 +DATA_RANGE= 1 25 !DATA_RANGE_FIXED_SCALE_FACTOR= 1900 6000 1.0 -SPOT_RANGE= 1 100 +SPOT_RANGE= 1 25 !BACKGROUND_RANGE= 1 11 !REFERENCE_DATA_SET= diff --git a/tests/xds_neggia/XDS.INP b/tests/xds_neggia/XDS.INP new file mode 100644 index 00000000..6b921ff0 --- /dev/null +++ b/tests/xds_neggia/XDS.INP @@ -0,0 +1,73 @@ + +!Settings for EIGER16M @ X06SA SLS +!job control +JOB=XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT + +MAXIMUM_NUMBER_OF_JOBS=1 +MAXIMUM_NUMBER_OF_PROCESSORS=1 + +! for this experiment: +ORGX= 1097 +ORGY= 1130 +DETECTOR_DISTANCE= 75 +OSCILLATION_RANGE= 0.088 +X-RAY_WAVELENGTH= 1.0 +NAME_TEMPLATE_OF_DATA_FRAMES=writing_test_??????.h5 +DATA_RANGE= 1 25 +!DATA_RANGE_FIXED_SCALE_FACTOR= 1900 6000 1.0 +SPOT_RANGE= 1 25 + +!BACKGROUND_RANGE= 1 11 +!REFERENCE_DATA_SET= + +SPACE_GROUP_NUMBER=96 +UNIT_CELL_CONSTANTS= 78.95 78.95 36.91 90.000 90.000 90.000 +!REIDX= 0 0 -1 0 0 -1 0 0 -1 0 0 0 + +REFINE(IDXREF)=BEAM AXIS ORIENTATION POSITION !CELL +REFINE(INTEGRATE)=POSITION BEAM ORIENTATION CELL AXIS +REFINE(CORRECT)=POSITION BEAM ORIENTATION CELL AXIS + +FRIEDEL'S_LAW=FALSE +STRICT_ABSORPTION_CORRECTION=FALSE ! but read Tips_and_Tricks in XDSwiki + +! parameters with changes wrt default values: +TRUSTED_REGION=0.00 1.21 +VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS=4000. 30000. +!MINIMUM_ZETA=0.05 +CORRECTIONS=DECAY MODULATION ABSORP !default value +INCLUDE_RESOLUTION_RANGE=50 0.0 + +! parameters specifically for this detector and beamline: +DETECTOR=EIGER NX=2068 NY=2164 QX=0.075 QY=0.075 !EIGER 16M +! Defines path to the Dectris neggia library (to use H5ToXds comment this line out) +LIB=/opt/xds/dectris-neggia.so + +MINIMUM_VALID_PIXEL_VALUE=0 OVERLOAD=30000 +DIRECTION_OF_DETECTOR_X-AXIS=1 0 0 +DIRECTION_OF_DETECTOR_Y-AXIS=0 1 0 +INCIDENT_BEAM_DIRECTION=0 0 1 +ROTATION_AXIS=1 0 0 +FRACTION_OF_POLARIZATION=0.99 +POLARIZATION_PLANE_NORMAL=0 1 0 +SENSOR_THICKNESS=0.32 +!EXCLUSION OF HORIZONTAL DEAD AREAS OF THE EIGER 16M DETECTOR + ONE PIXEL ON EACH SIDE +! UNTRUSTED_RECTANGLE= 0 4151 513 553 +! UNTRUSTED_RECTANGLE= 0 4151 1064 1104 +! UNTRUSTED_RECTANGLE= 0 4151 1615 1655 +! UNTRUSTED_RECTANGLE= 0 4151 2166 2206 +! UNTRUSTED_RECTANGLE= 0 4151 2717 2757 +! UNTRUSTED_RECTANGLE= 0 4151 3268 3308 +! UNTRUSTED_RECTANGLE= 0 4151 3819 3859 +!EXCLUSION OF VERTICAL DEAD AREAS OF THE EIGER 16M DETECTOR + ONE PIXEL ON EACH SIDE +! UNTRUSTED_RECTANGLE= 1029 1042 0 4372 +! UNTRUSTED_RECTANGLE= 2069 2082 0 4372 +! UNTRUSTED_RECTANGLE= 3109 3122 0 4372 + +UNTRUSTED_RECTANGLE=1036 2017 1082 1637 +NUMBER_OF_PROFILE_GRID_POINTS_ALONG_ALPHA/BETA=21 !used by: INTEGRATE +!MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT= +MINIMUM_FRACTION_OF_INDEXED_SPOTS=0.2 +SEPMIN=4 CLUSTER_RADIUS=2 +STRONG_PIXEL=40 + diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index d7165eb2..53cd8b8b 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -12,12 +12,14 @@ HDF5DataFile::HDF5DataFile(const std::string &in_filename, const std::vector& in_rad_int_bin_to_q, + int32_t in_image_low, size_t in_max_spots) { xpixel = 0; ypixel = 0; max_image_number = 0; nimages = 0; filename = in_filename; + image_low = in_image_low; plugins.emplace_back(std::make_unique()); plugins.emplace_back(std::make_unique(in_rad_int_bin_to_q)); @@ -37,6 +39,12 @@ HDF5DataFile::~HDF5DataFile() { for (auto &p: plugins) p->WriteFinal(*data_file); + if (data_set) { + data_set + ->Attr("image_nr_low", (int32_t) (image_low + 1)) + .Attr("image_nr_high", (int32_t) (image_low + 1 + max_image_number)); + data_set.reset(); + } data_file.reset(); std::string old_filename = filename + ".tmp"; std::rename(old_filename.c_str(), filename.c_str()); @@ -67,7 +75,6 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) { HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel}); data_set = std::make_unique(*data_file, "/entry/data/data", data_type, data_space, dcpl); - for (auto &p: plugins) p->OpenFile(*data_file, msg); } diff --git a/writer/HDF5DataFile.h b/writer/HDF5DataFile.h index 8558d51d..95ab1def 100644 --- a/writer/HDF5DataFile.h +++ b/writer/HDF5DataFile.h @@ -39,9 +39,12 @@ class HDF5DataFile { std::vector exptime; std::vector number; + int32_t image_low; + void CreateFile(const DataMessage& msg); public: - HDF5DataFile(const std::string& name, const std::vector& rad_int_bin_to_q, size_t max_spots = 0); + HDF5DataFile(const std::string& name, const std::vector& rad_int_bin_to_q, + int32_t image_low, size_t max_spots = 0); ~HDF5DataFile(); void Write(const DataMessage& msg, uint64_t image_number); diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index aa1f8f13..2256f214 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -83,11 +83,6 @@ void HDF5Metadata::Detector(HDF5File *hdf5_file, const StartMessage &start, cons SaveScalar(group, "acquisition_type", "triggered"); SaveScalar(group, "countrate_correction_applied", start.countrate_correction_enabled); SaveScalar(group, "number_of_cycles", start.summation); - // HDF5Group(group, "geometry").NXClass("NXgeometry"); - - // DIALS likes to have this soft link - H5Lcreate_soft("/entry/data/data", group.GetID(), "data", - H5P_DEFAULT, H5P_DEFAULT); HDF5Group det_specific(group, "detectorSpecific"); det_specific.NXClass("NXcollection"); @@ -219,8 +214,10 @@ void HDF5Metadata::SaveCBORImage(HDF5File *hdf5_file, const std::string &hdf5_pa HDF5DataType data_type(image.pixel_depth_bytes, image.pixel_is_signed); HDF5Dcpl dcpl; - dcpl.SetCompression(image.algorithm, H5Tget_size(data_type.GetID()), 0); - dcpl.SetChunking(dims); + if (image.algorithm != CompressionAlgorithm::NO_COMPRESSION) { + dcpl.SetCompression(image.algorithm, H5Tget_size(data_type.GetID()), 0); + dcpl.SetChunking(dims); + } HDF5DataSpace data_space(dims); auto dataset = std::make_unique(*hdf5_file, hdf5_path, data_type, data_space, dcpl); @@ -233,9 +230,7 @@ void HDF5Metadata::SaveCBORImage(HDF5File *hdf5_file, const std::string &hdf5_pa void HDF5Metadata::Calibration(HDF5File *hdf5_file, const StartMessage &start) { if (!start.pixel_mask.empty()) { - SaveCBORImage(hdf5_file, "/entry/instrument/detector/pixel_mask", start.pixel_mask[0]); - hdf5_file->HardLink("/entry/instrument/detector/pixel_mask", - "/entry/instrument/detector/detectorSpecific/pixel_mask"); + SaveCBORImage(hdf5_file, "/entry/instrument/detector/detectorSpecific/pixel_mask", start.pixel_mask[0]); } if (!start.gain_file_names.empty()) @@ -247,8 +242,6 @@ void HDF5Metadata::Calibration(HDF5File *hdf5_file, const StartMessage &start) { void HDF5Metadata::LinkToData(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) { hsize_t total_images = end.max_image_number; - hsize_t width = start.image_size_x; - hsize_t height = start.image_size_y; hsize_t images_per_file = start.images_per_file; hsize_t file_count = 0; if (start.images_per_file > 0) { @@ -257,42 +250,13 @@ void HDF5Metadata::LinkToData(HDF5File *hdf5_file, const StartMessage &start, co file_count++; } - if (total_images == 0) - return; - HDF5Group(*hdf5_file, "/entry/data").NXClass("NXdata"); - HDF5DataType data_type(start.pixel_bit_depth / 8, true); - HDF5DataSpace full_data_space({total_images, height, width}); - HDF5Dcpl dcpl; - - for (hsize_t file_id = 0; file_id < file_count; file_id++) { - - hsize_t images_in_file = images_per_file; - if (file_id == file_count - 1) - images_in_file = total_images - (file_count - 1) * images_per_file; - - HDF5DataSpace src_data_space({images_in_file, height, width}); - HDF5DataSpace virtual_data_space({total_images, height, width}); - virtual_data_space.SelectHyperslab({file_id * images_per_file, 0, 0},{images_in_file, height, width}); - dcpl.SetVirtual(DataFileName(start.file_prefix, file_id), - "/entry/data/data",src_data_space, virtual_data_space); + for (uint32_t file_id = 0; file_id < file_count; file_id++) { + char buff[32]; + snprintf(buff,32,"/entry/data/data_%06d", file_id+1); + hdf5_file->ExternalLink(DataFileName(start.file_prefix, file_id), "/entry/data/data", std::string(buff)); } - - if (start.pixel_bit_depth == 16) - dcpl.SetFillValue16(INT16_MIN); - else - dcpl.SetFillValue32(INT32_MIN); - - HDF5DataSet dataset(*hdf5_file, "/entry/data/data", data_type, full_data_space, dcpl); - dataset.Attr("image_nr_low", (int32_t) 1).Attr("image_nr_high", (int32_t) total_images); - - /* - if (experiment.GetDetectorMode() == DetectorMode::Conversion) - dataset.Units("photon"); - else - dataset.Units("ADU"); - */ } std::string HDF5Metadata::DataFileName(const std::string &prefix, int64_t file_number) { @@ -303,7 +267,7 @@ std::string HDF5Metadata::DataFileName(const std::string &prefix, int64_t file_n throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Format doesn't allow for 1 million or more files"); else - return fmt::format("{:s}_data_{:06d}.h5", prefix, file_number); + return fmt::format("{:s}_data_{:06d}.h5", prefix, file_number + 1); } void HDF5Metadata::AzimuthalIntegration(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) { diff --git a/writer/HDF5Objects.cpp b/writer/HDF5Objects.cpp index 012d93b2..a45ed06a 100644 --- a/writer/HDF5Objects.cpp +++ b/writer/HDF5Objects.cpp @@ -398,9 +398,7 @@ HDF5Group::~HDF5Group() { } HDF5File::HDF5File(const std::string& filename) : HDF5Object() { - HDF5Fapl fapl; - fapl.SetVersionTo1p10orNewer(); - id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl.GetID()); + id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); if (id < 0) throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot open/create data HDF5 file " + filename); } diff --git a/writer/HDF5Sum.cpp b/writer/HDF5Sum.cpp index 7cdc4f1d..70028239 100644 --- a/writer/HDF5Sum.cpp +++ b/writer/HDF5Sum.cpp @@ -43,7 +43,7 @@ int main(int argc, char **argv) { std::cout << "Summing dataset of " << nimages_in << " to " << nimages_out << " images" << std::endl; - HDF5DataFile dataFile(std::string(argv[2]), std::vector(), 1); + HDF5DataFile dataFile(std::string(argv[2]), std::vector(), nimages_out, 1); std::vector dim = {summation_factor, height, width}; diff --git a/writer/HDF5Writer.cpp b/writer/HDF5Writer.cpp index b78612b0..f1ba8920 100644 --- a/writer/HDF5Writer.cpp +++ b/writer/HDF5Writer.cpp @@ -27,7 +27,9 @@ void HDF5Writer::Write(const DataMessage& message) { if (!files[file_number]) files[file_number] = std::make_unique(HDF5Metadata::DataFileName(file_prefix, file_number), - az_int_bin_to_q, max_spot_count); + az_int_bin_to_q, + file_number * images_per_file, + max_spot_count); // Ignore zero size images if (message.image.size > 0) files[file_number]->Write(message, image_number);