mx/Jungfraujoch
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 37 Wiki Activity

Merge branch '2410-1.0.0.rc-18.fixes' into 'main'

Browse Source 
Fixes to version 1.0.0-rc.18

See merge request jungfraujoch/nextgendcu!88
This commit is contained in:
leonarski_f
2024-10-17 16:29:39 +02:00
parent 3d9c93c32a c9eb2ca65d
commit e6eb7af5ae
6 changed files with 136 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
frontend/src/components/DataCollection.tsx
View File

@@ -3,7 +3,7 @@ import React from 'react';
import Paper from '@mui/material/Paper';
import {Alert, SnackbarCloseReason, Stack, TextField} from "@mui/material";
import NumberTextField from "./NumberTextField";
import {ApiError, dataset_settings, DefaultService} from "../openapi";
import {ApiError, dataset_settings, DefaultService, OpenAPI} from "../openapi";
import Button from "@mui/material/Button";
import Snackbar from '@mui/material/Snackbar';
@@ -50,27 +50,42 @@ class DataCollection extends React.Component<MyProps, MyState> {
start_success: false
}
startButton = () => {
DefaultService.postStart(this.state.s)
.then(() => {
this.setState({
snackbar_open: true,
start_success: true
});
}).catch(error => {
let err : string = "Unknown error";
if (error instanceof ApiError)
err = `${error.status}: ${error.statusText}`
this.setState({
snackbar_open: true,
start_success: false,
start_error: err
});
printError = (msg: string) => {
this.setState({
snackbar_open: true,
start_success: false,
start_error: msg
});
}
handleResponse = (response : Response) => {
if (response.ok) {
this.setState({snackbar_open: true, start_success: true});
} else {
if (response.status == 404)
this.printError("404: Service not found");
else if (response.status == 400)
this.printError("400: Input parsing or validation error");
else if (response.status == 500) {
response.text().then((val: string) => {
try {
this.printError(JSON.parse(val).msg);
} catch (e) {
this.printError("500: Unknown error");
}
});
}
}
}
startButton = () => {
const url = OpenAPI.BASE + "/start";
fetch(url, {
method: "POST",
body: JSON.stringify(this.state.s)
}).then(this.handleResponse);
}
error() : boolean {
return this.state.beam_x_pxl_err
|| this.state.beam_y_pxl_err
@@ -226,7 +241,7 @@ class DataCollection extends React.Component<MyProps, MyState> {
Start
</Button>
<Snackbar open={this.state.snackbar_open}
autoHideDuration={6000}
autoHideDuration={10000}
onClose={this.handleClose}>
<Alert
onClose={this.handleClose}

116
receiver/JFJochReceiver.cpp
View File

@@ -180,56 +180,44 @@ void JFJochReceiver::SendStartMessage(const PixelMask &pixel_mask) {
image_pusher.StartDataCollection(message);
}
void JFJochReceiver::SendPedestal(const std::string &prefix, const std::vector<uint8_t> &v, int gain, int sc) {
size_t xpixel = RAW_MODULE_COLS;
size_t ypixel = experiment.GetModulesNum() * RAW_MODULE_LINES;
std::string channel;
if (experiment.GetStorageCellNumber() > 1)
channel = fmt::format("{:s}_g{:d}_sc{:d}", prefix, gain, sc);
else
channel = fmt::format("{:s}_g{:d}", prefix, gain);
CompressedImage image{
.data = v.data(),
.size = v.size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = channel
};
image_pusher.SendCalibration(image);
}
void JFJochReceiver::SendCalibration() {
if ((calibration == nullptr) || !experiment.GetSaveCalibration() || !push_images_to_writer)
return;
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
size_t xpixel = RAW_MODULE_COLS;
size_t ypixel = experiment.GetModulesNum() * RAW_MODULE_LINES;
for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) {
for (int gain = 0; gain < 3; gain++) {
if (experiment.IsFixedGainG1() && (gain != 1))
continue;
auto v = compressor.Compress(calibration->GetPedestal(gain, sc));
std::string channel = "pedestal_G" + std::to_string(gain);
if (experiment.GetStorageCellNumber() > 1)
channel += "_sc" + std::to_string(sc);
CompressedImage image{
.data = v.data(),
.size = v.size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = channel
};
image_pusher.SendCalibration(image);
auto v_rms = compressor.Compress(calibration->GetPedestalRMS(gain, sc));
channel = "pedestalRMS_G" + std::to_string(gain);
if (experiment.GetStorageCellNumber() > 1)
channel += "_sc" + std::to_string(sc);
CompressedImage image_rms{
.data = v.data(),
.size = v.size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = channel
};
image_pusher.SendCalibration(image);
SendPedestal("pedestal", compressor.Compress(calibration->GetPedestal(gain, sc)), gain, sc);
SendPedestal("pedestal_rms", compressor.Compress(calibration->GetPedestalRMS(gain, sc)), gain, sc);
}
}
}
@@ -242,26 +230,7 @@ void JFJochReceiver::AcquireThread(uint16_t data_stream) {
}
try {
if (calibration != nullptr)
acquisition_device[data_stream].InitializeCalibration(experiment, *calibration);
size_t m_offset = experiment.GetFirstModuleOfDataStream(data_stream);
acquisition_device[data_stream].InitializeROIMap(experiment);
std::vector<float> tmp(RAW_MODULE_SIZE);
for (int m = 0; m < experiment.GetModulesNum(data_stream); m++) {
CalcSpotFinderResolutionMap(tmp.data(), experiment, m_offset + m);
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
if (user_mask_raw_coord[(m_offset + m) * RAW_MODULE_SIZE + i] != 0)
tmp[i] = 0.0f;
}
acquisition_device[data_stream].InitializeSpotFinderResolutionMap(tmp.data(), m);
CalcAzIntCorrRawCoord(tmp.data(), experiment, m_offset + m);
acquisition_device[data_stream].InitializeIntegrationMap(az_int_mapping.GetPixelToBinMappingRaw().data()
+ (m_offset + m) * RAW_MODULE_SIZE,
tmp.data(), m);
}
LoadCalibrationToFPGA(data_stream);
frame_transformation_ready.wait();
logger.Debug("Device thread {} start FPGA action", data_stream);
acquisition_device[data_stream].StartAction(experiment);
@@ -672,3 +641,30 @@ void JFJochReceiver::RetrievePedestal(std::vector<JFModulePedestal> &output) con
output.emplace_back(std::move(mp));
}
}
void JFJochReceiver::LoadCalibrationToFPGA(uint16_t data_stream) {
if (experiment.IsPedestalRun())
return; // No calibration loaded for pedestal
if (calibration != nullptr)
acquisition_device[data_stream].InitializeCalibration(experiment, *calibration);
// Initialize roi_map
acquisition_device[data_stream].InitializeROIMap(experiment);
size_t m0 = experiment.GetFirstModuleOfDataStream(data_stream);
std::vector<float> tmp(RAW_MODULE_SIZE);
for (int m = 0; m < experiment.GetModulesNum(data_stream); m++) {
// Mask pixels for spot finding based on resolution and user mask
CalcSpotFinderResolutionMap(tmp.data(), experiment, m0 + m);
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
if (user_mask_raw_coord[(m0 + m) * RAW_MODULE_SIZE + i] != 0)
tmp[i] = 0.0f;
}
acquisition_device[data_stream].InitializeSpotFinderResolutionMap(tmp.data(), m);
CalcAzIntCorrRawCoord(tmp.data(), experiment, m0 + m);
acquisition_device[data_stream].InitializeIntegrationMap(az_int_mapping.GetPixelToBinMappingRaw().data()
+ (m0 + m) * RAW_MODULE_SIZE, tmp.data(), m);
}
}

3
receiver/JFJochReceiver.h
View File

@@ -111,6 +111,8 @@ class JFJochReceiver {
std::vector<uint32_t> user_mask_raw_coord;
void AcquireThread(uint16_t data_stream);
void LoadCalibrationToFPGA(uint16_t data_stream);
void FrameTransformationThread(uint32_t threadid);
void MeasurePedestalThread(uint16_t data_stream,
uint16_t module_number,
@@ -124,6 +126,7 @@ class JFJochReceiver {
void UpdateMaxDelay(uint64_t delay);
void SendStartMessage(const PixelMask &pixel_mask);
void SendCalibration();
void SendPedestal(const std::string &prefix, const std::vector<uint8_t> &v, int gain, int sc);
void RetrievePedestal(std::vector<JFModulePedestal> &output) const;
JFJochReceiverStatus GetStatus() const;
public:

54
tests/JFJochReceiverIntegrationTest.cpp
View File

@@ -784,57 +784,3 @@ TEST_CASE("JFJochReceiverTest_EIGER_conversion", "[JFJochReceiver]") {
REQUIRE(!output.status.cancelled);
}
#include "../image_pusher/HDF5FilePusher.h"
TEST_CASE("JFJochIntegrationTest_HDF5FilePusher", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_HDF5FilePusher");
RegisterHDF5Filter();
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2)
.FilePrefix("lyso_test_HDF5FilePusher").JungfrauConvPhotonCnt(false)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
PixelMask pixel_mask(experiment);
// Load example image
HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
HDF5DataSet dataset(data, "/entry/data/data");
HDF5DataSpace file_space(dataset);
REQUIRE(file_space.GetDimensions()[2] == experiment.GetXPixelsNum());
REQUIRE(file_space.GetDimensions()[1] == experiment.GetYPixelsNum());
std::vector<int16_t> image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
std::vector<hsize_t> start = {4,0,0};
std::vector<hsize_t> file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
dataset.ReadVector(image_conv, start, file_size);
std::vector<int16_t> image_raw_geom(experiment.GetModulesNum() * RAW_MODULE_SIZE);
ConvertedToRawGeometry(experiment, image_raw_geom.data(), image_conv.data());
logger.Info("Loaded image");
// Setup acquisition device
AcquisitionDeviceGroup aq_devices;
std::unique_ptr<HLSSimulatedDevice> test = std::make_unique<HLSSimulatedDevice>(0, 64);
for (int m = 0; m < experiment.GetModulesNum(); m++)
test->SetInternalGeneratorFrame((uint16_t *) image_raw_geom.data() + m * RAW_MODULE_SIZE, m);
aq_devices.Add(std::move(test));
HDF5FilePusher pusher;
JFJochReceiverService service(aq_devices, logger, pusher);
service.NumThreads(nthreads);
service.Start(experiment, pixel_mask, nullptr);
auto receiver_out = service.Stop();
CHECK(receiver_out.efficiency == 1.0);
CHECK(receiver_out.status.images_sent == experiment.GetImageNum());
CHECK(!receiver_out.status.cancelled);
}

36
tests/JFJochReceiverProcessingTest.cpp
View File

@@ -14,6 +14,7 @@
#include "../image_pusher/ZMQStream2Pusher.h"
#include "../writer/StreamWriter.h"
#include "../image_pusher/NonePusher.h"
#include "../image_pusher/HDF5FilePusher.h"
TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index");
@@ -289,7 +290,7 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_ROI", "[JFJochReceiver]") {
}
TEST_CASE("JFJochIntegrationTest_ZMQPreview", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_ZMQ_ROI");
Logger logger("JFJochIntegrationTest_ZMQPreview");
RegisterHDF5Filter();
@@ -335,3 +336,36 @@ TEST_CASE("JFJochIntegrationTest_ZMQPreview", "[JFJochReceiver]") {
REQUIRE(sub_socket.Receive(msg1, true));
REQUIRE(!sub_socket.Receive(msg1, true));
}
TEST_CASE("JFJochIntegrationTest_HDF5FilePusher", "[JFJochReceiver]") {
Logger logger("JFJochIntegrationTest_HDF5FilePusher");
RegisterHDF5Filter();
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetectorGeometry(8,2,8,36));
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true)
.FilePrefix("HDF5FilePusherTest").JungfrauConvPhotonCnt(true)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).PhotonEnergy_keV(12.4);
JFCalibration calibration(experiment);
PixelMask pixel_mask(experiment);
// Setup acquisition device
AcquisitionDeviceGroup aq_devices;
aq_devices.Add(std::make_unique<HLSSimulatedDevice>(0, 64));
HDF5FilePusher pusher;
JFJochReceiverService service(aq_devices, logger, pusher);
service.NumThreads(nthreads);
service.Start(experiment, pixel_mask, &calibration);
auto receiver_out = service.Stop();
CHECK(receiver_out.efficiency == 1.0);
CHECK(receiver_out.status.images_collected == 5);
CHECK(receiver_out.status.images_sent == 5);
CHECK(!receiver_out.status.cancelled);
}

8
writer/HDF5Writer.cpp
View File

@@ -4,6 +4,8 @@
#include <nlohmann/json.hpp>
#include "MakeDirectory.h"
#include "../common/CheckPath.h"
#include "../common/Logger.h"
#include "../common/JFJochException.h"
HDF5Writer::HDF5Writer(const StartMessage &request)
: start_message(request) {
@@ -116,7 +118,11 @@ std::optional<std::string> HDF5Writer::GetZMQAddr() {
void HDF5Writer::Write(const CompressedImage &msg) {
if (master_file) {
std::lock_guard<std::mutex> lock(hdf5_mutex);
master_file->WriteCalibration(msg);
try {
master_file->WriteCalibration(msg);
} catch (const JFJochException &e) {
spdlog::error("Calibration {} not written {}", msg.channel, e.what());
}
}
}