diff --git a/frontend/src/components/DataCollection.tsx b/frontend/src/components/DataCollection.tsx index b00faf60..1e1c93bd 100644 --- a/frontend/src/components/DataCollection.tsx +++ b/frontend/src/components/DataCollection.tsx @@ -3,7 +3,7 @@ import React from 'react'; import Paper from '@mui/material/Paper'; import {Alert, SnackbarCloseReason, Stack, TextField} from "@mui/material"; import NumberTextField from "./NumberTextField"; -import {ApiError, dataset_settings, DefaultService} from "../openapi"; +import {ApiError, dataset_settings, DefaultService, OpenAPI} from "../openapi"; import Button from "@mui/material/Button"; import Snackbar from '@mui/material/Snackbar'; @@ -50,27 +50,42 @@ class DataCollection extends React.Component { start_success: false } - startButton = () => { - DefaultService.postStart(this.state.s) - .then(() => { - this.setState({ - snackbar_open: true, - start_success: true - }); - }).catch(error => { - let err : string = "Unknown error"; - - if (error instanceof ApiError) - err = `${error.status}: ${error.statusText}` - - this.setState({ - snackbar_open: true, - start_success: false, - start_error: err - }); + printError = (msg: string) => { + this.setState({ + snackbar_open: true, + start_success: false, + start_error: msg }); } + handleResponse = (response : Response) => { + if (response.ok) { + this.setState({snackbar_open: true, start_success: true}); + } else { + if (response.status == 404) + this.printError("404: Service not found"); + else if (response.status == 400) + this.printError("400: Input parsing or validation error"); + else if (response.status == 500) { + response.text().then((val: string) => { + try { + this.printError(JSON.parse(val).msg); + } catch (e) { + this.printError("500: Unknown error"); + } + }); + } + } + } + + startButton = () => { + const url = OpenAPI.BASE + "/start"; + fetch(url, { + method: "POST", + body: JSON.stringify(this.state.s) + }).then(this.handleResponse); + } + error() : boolean { return this.state.beam_x_pxl_err || this.state.beam_y_pxl_err @@ -226,7 +241,7 @@ class DataCollection extends React.Component { Start &v, int gain, int sc) { + size_t xpixel = RAW_MODULE_COLS; + size_t ypixel = experiment.GetModulesNum() * RAW_MODULE_LINES; + + std::string channel; + + if (experiment.GetStorageCellNumber() > 1) + channel = fmt::format("{:s}_g{:d}_sc{:d}", prefix, gain, sc); + else + channel = fmt::format("{:s}_g{:d}", prefix, gain); + + CompressedImage image{ + .data = v.data(), + .size = v.size(), + .xpixel = (size_t) xpixel, + .ypixel = (size_t) ypixel, + .pixel_depth_bytes = 2, + .pixel_is_signed = false, + .pixel_is_float = false, + .algorithm = CompressionAlgorithm::BSHUF_LZ4, + .channel = channel + }; + + image_pusher.SendCalibration(image); +} + void JFJochReceiver::SendCalibration() { if ((calibration == nullptr) || !experiment.GetSaveCalibration() || !push_images_to_writer) return; JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); - size_t xpixel = RAW_MODULE_COLS; - size_t ypixel = experiment.GetModulesNum() * RAW_MODULE_LINES; for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) { for (int gain = 0; gain < 3; gain++) { if (experiment.IsFixedGainG1() && (gain != 1)) continue; - - auto v = compressor.Compress(calibration->GetPedestal(gain, sc)); - std::string channel = "pedestal_G" + std::to_string(gain); - - if (experiment.GetStorageCellNumber() > 1) - channel += "_sc" + std::to_string(sc); - - CompressedImage image{ - .data = v.data(), - .size = v.size(), - .xpixel = (size_t) xpixel, - .ypixel = (size_t) ypixel, - .pixel_depth_bytes = 2, - .pixel_is_signed = false, - .pixel_is_float = false, - .algorithm = CompressionAlgorithm::BSHUF_LZ4, - .channel = channel - }; - image_pusher.SendCalibration(image); - - auto v_rms = compressor.Compress(calibration->GetPedestalRMS(gain, sc)); - channel = "pedestalRMS_G" + std::to_string(gain); - - if (experiment.GetStorageCellNumber() > 1) - channel += "_sc" + std::to_string(sc); - - CompressedImage image_rms{ - .data = v.data(), - .size = v.size(), - .xpixel = (size_t) xpixel, - .ypixel = (size_t) ypixel, - .pixel_depth_bytes = 2, - .pixel_is_signed = false, - .pixel_is_float = false, - .algorithm = CompressionAlgorithm::BSHUF_LZ4, - .channel = channel - }; - image_pusher.SendCalibration(image); + SendPedestal("pedestal", compressor.Compress(calibration->GetPedestal(gain, sc)), gain, sc); + SendPedestal("pedestal_rms", compressor.Compress(calibration->GetPedestalRMS(gain, sc)), gain, sc); } } } @@ -242,26 +230,7 @@ void JFJochReceiver::AcquireThread(uint16_t data_stream) { } try { - if (calibration != nullptr) - acquisition_device[data_stream].InitializeCalibration(experiment, *calibration); - - size_t m_offset = experiment.GetFirstModuleOfDataStream(data_stream); - acquisition_device[data_stream].InitializeROIMap(experiment); - - std::vector tmp(RAW_MODULE_SIZE); - for (int m = 0; m < experiment.GetModulesNum(data_stream); m++) { - CalcSpotFinderResolutionMap(tmp.data(), experiment, m_offset + m); - for (int i = 0; i < RAW_MODULE_SIZE; i++) { - if (user_mask_raw_coord[(m_offset + m) * RAW_MODULE_SIZE + i] != 0) - tmp[i] = 0.0f; - } - acquisition_device[data_stream].InitializeSpotFinderResolutionMap(tmp.data(), m); - - CalcAzIntCorrRawCoord(tmp.data(), experiment, m_offset + m); - acquisition_device[data_stream].InitializeIntegrationMap(az_int_mapping.GetPixelToBinMappingRaw().data() - + (m_offset + m) * RAW_MODULE_SIZE, - tmp.data(), m); - } + LoadCalibrationToFPGA(data_stream); frame_transformation_ready.wait(); logger.Debug("Device thread {} start FPGA action", data_stream); acquisition_device[data_stream].StartAction(experiment); @@ -672,3 +641,30 @@ void JFJochReceiver::RetrievePedestal(std::vector &output) con output.emplace_back(std::move(mp)); } } + +void JFJochReceiver::LoadCalibrationToFPGA(uint16_t data_stream) { + if (experiment.IsPedestalRun()) + return; // No calibration loaded for pedestal + + if (calibration != nullptr) + acquisition_device[data_stream].InitializeCalibration(experiment, *calibration); + + // Initialize roi_map + acquisition_device[data_stream].InitializeROIMap(experiment); + + size_t m0 = experiment.GetFirstModuleOfDataStream(data_stream); + std::vector tmp(RAW_MODULE_SIZE); + for (int m = 0; m < experiment.GetModulesNum(data_stream); m++) { + // Mask pixels for spot finding based on resolution and user mask + CalcSpotFinderResolutionMap(tmp.data(), experiment, m0 + m); + for (int i = 0; i < RAW_MODULE_SIZE; i++) { + if (user_mask_raw_coord[(m0 + m) * RAW_MODULE_SIZE + i] != 0) + tmp[i] = 0.0f; + } + acquisition_device[data_stream].InitializeSpotFinderResolutionMap(tmp.data(), m); + + CalcAzIntCorrRawCoord(tmp.data(), experiment, m0 + m); + acquisition_device[data_stream].InitializeIntegrationMap(az_int_mapping.GetPixelToBinMappingRaw().data() + + (m0 + m) * RAW_MODULE_SIZE, tmp.data(), m); + } +} diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 9bafec14..9baa9c48 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -111,6 +111,8 @@ class JFJochReceiver { std::vector user_mask_raw_coord; void AcquireThread(uint16_t data_stream); + void LoadCalibrationToFPGA(uint16_t data_stream); + void FrameTransformationThread(uint32_t threadid); void MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, @@ -124,6 +126,7 @@ class JFJochReceiver { void UpdateMaxDelay(uint64_t delay); void SendStartMessage(const PixelMask &pixel_mask); void SendCalibration(); + void SendPedestal(const std::string &prefix, const std::vector &v, int gain, int sc); void RetrievePedestal(std::vector &output) const; JFJochReceiverStatus GetStatus() const; public: diff --git a/tests/JFJochReceiverIntegrationTest.cpp b/tests/JFJochReceiverIntegrationTest.cpp index 88ea7658..1ade4f44 100644 --- a/tests/JFJochReceiverIntegrationTest.cpp +++ b/tests/JFJochReceiverIntegrationTest.cpp @@ -784,57 +784,3 @@ TEST_CASE("JFJochReceiverTest_EIGER_conversion", "[JFJochReceiver]") { REQUIRE(!output.status.cancelled); } - -#include "../image_pusher/HDF5FilePusher.h" - -TEST_CASE("JFJochIntegrationTest_HDF5FilePusher", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_HDF5FilePusher"); - - RegisterHDF5Filter(); - - const uint16_t nthreads = 4; - - DiffractionExperiment experiment(DetectorGeometry(8,2,8,36)); - experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2) - .FilePrefix("lyso_test_HDF5FilePusher").JungfrauConvPhotonCnt(false) - .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4) - .SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90}); - PixelMask pixel_mask(experiment); - - // Load example image - HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); - HDF5DataSet dataset(data, "/entry/data/data"); - HDF5DataSpace file_space(dataset); - - REQUIRE(file_space.GetDimensions()[2] == experiment.GetXPixelsNum()); - REQUIRE(file_space.GetDimensions()[1] == experiment.GetYPixelsNum()); - std::vector image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - std::vector start = {4,0,0}; - std::vector file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; - dataset.ReadVector(image_conv, start, file_size); - - std::vector image_raw_geom(experiment.GetModulesNum() * RAW_MODULE_SIZE); - ConvertedToRawGeometry(experiment, image_raw_geom.data(), image_conv.data()); - logger.Info("Loaded image"); - - // Setup acquisition device - AcquisitionDeviceGroup aq_devices; - std::unique_ptr test = std::make_unique(0, 64); - for (int m = 0; m < experiment.GetModulesNum(); m++) - test->SetInternalGeneratorFrame((uint16_t *) image_raw_geom.data() + m * RAW_MODULE_SIZE, m); - - aq_devices.Add(std::move(test)); - - HDF5FilePusher pusher; - - JFJochReceiverService service(aq_devices, logger, pusher); - service.NumThreads(nthreads); - - service.Start(experiment, pixel_mask, nullptr); - auto receiver_out = service.Stop(); - - CHECK(receiver_out.efficiency == 1.0); - CHECK(receiver_out.status.images_sent == experiment.GetImageNum()); - CHECK(!receiver_out.status.cancelled); -} diff --git a/tests/JFJochReceiverProcessingTest.cpp b/tests/JFJochReceiverProcessingTest.cpp index 74f3497b..7a3e7c8f 100644 --- a/tests/JFJochReceiverProcessingTest.cpp +++ b/tests/JFJochReceiverProcessingTest.cpp @@ -14,6 +14,7 @@ #include "../image_pusher/ZMQStream2Pusher.h" #include "../writer/StreamWriter.h" #include "../image_pusher/NonePusher.h" +#include "../image_pusher/HDF5FilePusher.h" TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index", "[JFJochReceiver]") { Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index"); @@ -289,7 +290,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_ROI", "[JFJochReceiver]") { } TEST_CASE("JFJochIntegrationTest_ZMQPreview", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_ROI"); + Logger logger("JFJochIntegrationTest_ZMQPreview"); RegisterHDF5Filter(); @@ -335,3 +336,36 @@ TEST_CASE("JFJochIntegrationTest_ZMQPreview", "[JFJochReceiver]") { REQUIRE(sub_socket.Receive(msg1, true)); REQUIRE(!sub_socket.Receive(msg1, true)); } + +TEST_CASE("JFJochIntegrationTest_HDF5FilePusher", "[JFJochReceiver]") { + Logger logger("JFJochIntegrationTest_HDF5FilePusher"); + + RegisterHDF5Filter(); + + const uint16_t nthreads = 4; + + DiffractionExperiment experiment(DetectorGeometry(8,2,8,36)); + experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true) + .FilePrefix("HDF5FilePusherTest").JungfrauConvPhotonCnt(true) + .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4); + + JFCalibration calibration(experiment); + PixelMask pixel_mask(experiment); + + // Setup acquisition device + AcquisitionDeviceGroup aq_devices; + aq_devices.Add(std::make_unique(0, 64)); + + HDF5FilePusher pusher; + + JFJochReceiverService service(aq_devices, logger, pusher); + service.NumThreads(nthreads); + + service.Start(experiment, pixel_mask, &calibration); + auto receiver_out = service.Stop(); + + CHECK(receiver_out.efficiency == 1.0); + CHECK(receiver_out.status.images_collected == 5); + CHECK(receiver_out.status.images_sent == 5); + CHECK(!receiver_out.status.cancelled); +} diff --git a/writer/HDF5Writer.cpp b/writer/HDF5Writer.cpp index 0c8b564d..9ef178b0 100644 --- a/writer/HDF5Writer.cpp +++ b/writer/HDF5Writer.cpp @@ -4,6 +4,8 @@ #include #include "MakeDirectory.h" #include "../common/CheckPath.h" +#include "../common/Logger.h" +#include "../common/JFJochException.h" HDF5Writer::HDF5Writer(const StartMessage &request) : start_message(request) { @@ -116,7 +118,11 @@ std::optional HDF5Writer::GetZMQAddr() { void HDF5Writer::Write(const CompressedImage &msg) { if (master_file) { std::lock_guard lock(hdf5_mutex); - master_file->WriteCalibration(msg); + try { + master_file->WriteCalibration(msg); + } catch (const JFJochException &e) { + spdlog::error("Calibration {} not written {}", msg.channel, e.what()); + } } }