diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0746516a..9dc517f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,9 +47,15 @@ build:x86:driver: - gcc - x86 needs: [] + artifacts: + paths: + - "jfjoch_driver_*.tar.gz" + expire_in: 1 week script: - cd fpga/pcie_driver - make + - bash pack.sh + - mv jfjoch_driver_*.tar.gz ../.. build:x86:vitis_hls: stage: build @@ -59,7 +65,8 @@ build:x86:vitis_hls: - x86 needs: [] rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "push" changes: - fpga/hls/* - fpga/hdl/* @@ -228,7 +235,8 @@ synthesis:vivado_pcie_100g: CXX: g++ allow_failure: true rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "push" changes: - fpga/hls/* - fpga/hdl/* @@ -259,7 +267,8 @@ synthesis:vivado_pcie_8x10g: CXX: g++ allow_failure: true rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "push" changes: - fpga/hls/* - fpga/hdl/* diff --git a/README.md b/README.md index 70094ad8..3bc4ee5e 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,13 @@ cd build cmake -DJFJOCH_WRITER_ONLY=ON .. make jfjoch ``` + +## Versions +**FPGA release** is as hexadecimal number indicted in the [jfjoch_fpga.h](fpga/pcie_driver/jfjoch_fpga.h) as JFJOCH_FPGA_RELEASE constant. +It is also included in the name of FPGA firmware file (.mcs) and kernel driver archive. This number indicated breaking changes in the FPGA firmware interface. +FPGA release has to be consistent between FPGA firmware, kernel driver and `jfjoch_broker` - both kernel driver and software won't work in case of version mismatch. +Commits to `main` branch with the same FPGA release version are OK to mix between components. + ## Web Frontend Jungfraujoch is equipped with React-based web frontend for user-friendly experience. Frontend has the following options: * Presenting current state of the detector @@ -104,6 +111,16 @@ Jungfraujoch is equipped with React-based web frontend for user-friendly experie Frontend is written in TypeScript. For details see [frontend_ui/](frontend_ui) directory. +Jungfraujoch Cmake scripts have an option to start frontend build with the following command: +``` +git submodule update --init --recursive +mkdir build +cd build +cmake .. +make frontend +``` +Contrary to standard CMake way, frontend will be built in "source" `frontend_ui/build` directory, not in `build/` subdirectory. + ## Tests Automated test routine is then accessible as `tests/CatchTest`. There are also benchmark routines: diff --git a/acquisition_device/AcquisitionDevice.h b/acquisition_device/AcquisitionDevice.h index 8c49541e..d395fab6 100644 --- a/acquisition_device/AcquisitionDevice.h +++ b/acquisition_device/AcquisitionDevice.h @@ -17,7 +17,7 @@ #include "AcquisitionCounters.h" #include "Completion.h" -#include "../fpga/include/jfjoch_fpga.h" +#include "../fpga/pcie_driver/jfjoch_fpga.h" #include "../common/NetworkAddressConvert.h" struct AcquisitionDeviceStatistics { diff --git a/acquisition_device/CMakeLists.txt b/acquisition_device/CMakeLists.txt index 5f8510ef..020a8f68 100644 --- a/acquisition_device/CMakeLists.txt +++ b/acquisition_device/CMakeLists.txt @@ -2,7 +2,7 @@ ADD_LIBRARY(JFJochAcquisitionDevice STATIC AcquisitionDevice.cpp AcquisitionDevice.h AcquisitionCounters.cpp AcquisitionCounters.h HLSSimulatedDevice.cpp HLSSimulatedDevice.h - Completion.cpp Completion.h ../fpga/include/jfjoch_fpga.h + Completion.cpp Completion.h ../fpga/pcie_driver/jfjoch_fpga.h PCIExpressDevice.cpp PCIExpressDevice.h FPGAAcquisitionDevice.cpp FPGAAcquisitionDevice.h AcquisitionDeviceGroup.cpp diff --git a/acquisition_device/FPGAAcquisitionDevice.h b/acquisition_device/FPGAAcquisitionDevice.h index a2b668d2..3ebfb9b4 100644 --- a/acquisition_device/FPGAAcquisitionDevice.h +++ b/acquisition_device/FPGAAcquisitionDevice.h @@ -4,7 +4,7 @@ #define JUNGFRAUJOCH_FPGAACQUISITIONDEVICE_H #include "AcquisitionDevice.h" -#include "../fpga/include/jfjoch_fpga.h" +#include "../fpga/pcie_driver/jfjoch_fpga.h" class FPGAAcquisitionDevice : public AcquisitionDevice { uint16_t data_collection_id = 0; diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index d78920d4..c45f854c 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -381,11 +381,8 @@ std::vector ParseStringArray(const nlohmann::json &input, const std return GET_STR_ARR(input, tag); } -std::string ParseString(const nlohmann::json &input, const std::string& tag) { - if (input.contains(tag)) - return GET_STR(input, tag); - else - return ""; +std::string ParseString(const nlohmann::json &input, const std::string& tag, const std::string &def) { + return GET_STR(input, tag, def); } int64_t ParseInt64(const nlohmann::json &input, const std::string& tag, int64_t def) { diff --git a/broker/JFJochBrokerParser.h b/broker/JFJochBrokerParser.h index 56617938..7cce97e7 100644 --- a/broker/JFJochBrokerParser.h +++ b/broker/JFJochBrokerParser.h @@ -17,7 +17,7 @@ void ParseFacilityConfiguration(const nlohmann::json &j, const std::string& tag, void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string& tag, AcquisitionDeviceGroup &aq_devices); std::vector ParseStringArray(const nlohmann::json &input, const std::string& tag); -std::string ParseString(const nlohmann::json &input, const std::string& tag); +std::string ParseString(const nlohmann::json &input, const std::string& tag, const std::string& def); int64_t ParseInt64(const nlohmann::json &input, const std::string& tag, int64_t def); int32_t ParseInt32(const nlohmann::json &input, const std::string& tag, int32_t def); diff --git a/broker/README.md b/broker/README.md new file mode 100644 index 00000000..bdf21713 --- /dev/null +++ b/broker/README.md @@ -0,0 +1,47 @@ +# Jungfraujoch service (jfjoch_broker) + +'jfjoch_broker' is the main service for the Jungfraujoch application. It is responsible for: + +* Providing user interface via HTTP and OpenAPI +* Configuring FPGA firmware +* Building images from FPGA output and forwarding the results over ZeroMQ + +Description of OpenAPI is presented in the `redoc-static.html` file. +Due to limitations of GitLab/GitHub the file cannot be viewed directly from the repository, but needs to be downloaded. +## Broker configuration +'jfjoch_broker' requires JSON configuration files. At the moment the format of the configuration file is not properly documented. +Till this happens, it is recommended to go through example files in the [etc/](../etc/). + +## Setting up a local test for Jungfraujoch +For development, it is possible to setup a local installation of Jungfraujoch. +This will work without FPGA installed in the computer and allows to test Jungfraujoch software layer, including +ZeroMQ streaming and file writing. There are few necessary steps: + +### Compile Jungfraujoch with frontend +``` +git submodule update --init --recursive +mkdir build +cd build +cmake .. +make jfjoch +make frontend +``` +### Start services +Start broker: +``` +cd build/broker +./jfjoch_broker ../../etc/broker_local.json 5232 +``` + +Start writer: +``` +cd build/writer +./jfjoch_writer tcp://127.0.0.1:5500 +``` +### Run tests +To run test a Python script is provided: +``` +cd tests/test_data +python jfjoch_broker_test.py +``` +The script will initialize Jungfraujoch, import test image and start data collection. diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index 14a0260f..a286edc8 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -13,6 +13,7 @@ #include "JFJochBrokerParser.h" #include "../frame_serialize/ZMQStream2PusherGroup.h" +#include "../frame_serialize/DumpCBORToFilePusher.h" static Pistache::Http::Endpoint *httpEndpoint; @@ -70,7 +71,7 @@ int main (int argc, char **argv) { } std::unique_ptr receiver; - std::unique_ptr image_pusher; + std::unique_ptr image_pusher; ZMQContext context; @@ -82,17 +83,24 @@ int main (int argc, char **argv) { if (aq_devices.size() > 0) { experiment.DataStreams(aq_devices.size()); - int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100); - int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1); + std::string pusher_type = ParseString(input, "stream_type", "zmq"); + if (pusher_type == "zmq") { + int32_t zmq_send_watermark = ParseInt32(input, "zmq_send_watermark", 100); + int32_t zmq_send_buffer_size = ParseInt32(input, "zmq_send_buffer_size", -1); - image_pusher = std::make_unique(ParseStringArray(input, "zmq_image_addr"), - zmq_send_watermark, - zmq_send_buffer_size); + image_pusher = std::make_unique(ParseStringArray(input, "zmq_image_addr"), + zmq_send_watermark, + zmq_send_buffer_size); + } else if (pusher_type == "dump_cbor") { + image_pusher = std::make_unique(); + } else + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "stream_type allowed: zmq (default), dump_cbor"); int32_t send_buffer_size_MiB = ParseInt32(input, "send_buffer_size_MiB", 2048); receiver = std::make_unique(aq_devices, logger, *image_pusher, send_buffer_size_MiB); - std::string numa_policy = ParseString(input, "numa_policy"); + std::string numa_policy = ParseString(input, "numa_policy", ""); if (!numa_policy.empty()) receiver->NUMAPolicy(numa_policy); diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index ac58080b..70b94333 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -42,7 +42,7 @@ ADD_LIBRARY( JFJochCommon STATIC ADUHistogram.cpp ADUHistogram.h RawToConvertedGeometryCore.h Plot.h - ../fpga/include/jfjoch_fpga.h + ../fpga/pcie_driver/jfjoch_fpga.h ZMQWrappers.cpp ZMQWrappers.h DatasetSettings.cpp DatasetSettings.h ROIMap.cpp ROIMap.h diff --git a/common/Definitions.h b/common/Definitions.h index 2e376d46..e315598e 100644 --- a/common/Definitions.h +++ b/common/Definitions.h @@ -3,7 +3,7 @@ #ifndef DEFINITIONS_H #define DEFINITIONS_H -#include "../fpga/include/jfjoch_fpga.h" +#include "../fpga/pcie_driver/jfjoch_fpga.h" #define WVL_1A_IN_KEV 12.39854f diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 4fa7128f..b4a0e88e 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -8,7 +8,7 @@ #include "DiffractionExperiment.h" #include "JFJochException.h" #include "RawToConvertedGeometry.h" -#include "../fpga/include/jfjoch_fpga.h" +#include "../fpga/pcie_driver/jfjoch_fpga.h" using namespace std::literals::chrono_literals; @@ -66,7 +66,7 @@ DiffractionExperiment::DiffractionExperiment(const DetectorSetup& det_setup) mode = DetectorMode::Conversion; - max_spot_count = 100; + max_spot_count = MAX_SPOT_COUNT; } // setter functions diff --git a/fpga/README.md b/fpga/README.md index 100bc35c..454de03e 100644 --- a/fpga/README.md +++ b/fpga/README.md @@ -135,7 +135,7 @@ This number is incremented after each change of functionality of the card or int This ensures consistency between the FPGA card, driver and user application. Changes within the design (e.g. size of FIFOs), that are invisible to interactions with host do not require change in release number. -To check release number, look for constant `RELEASE_NUMBER` in [include/jfjoch_fpga.h](include/jfjoch_fpga.h) header file. +To check release number, look for constant `RELEASE_NUMBER` in [pcie_driver/jfjoch_fpga.h](pcie_driver/jfjoch_fpga.h) header file. For FPGA design, release number is also included in the generated bitstream name. In case there is mismatch in release number between card and kernel driver, the latter will not create the character device and return error (check `dmesg`). diff --git a/fpga/hls/CMakeLists.txt b/fpga/hls/CMakeLists.txt index d0e68273..00e6d7e5 100644 --- a/fpga/hls/CMakeLists.txt +++ b/fpga/hls/CMakeLists.txt @@ -3,7 +3,7 @@ ADD_LIBRARY( JFJochHLSSimulation STATIC data_collection_fsm.cpp timer.cpp hls_jfjoch.h - ../include/jfjoch_fpga.h + ../pcie_driver/jfjoch_fpga.h load_calibration.cpp host_writer.cpp ethernet.cpp @@ -65,7 +65,7 @@ FUNCTION( MAKE_HLS_MODULE FUNCTION_NAME SRC_FILE TB_FILE) COMMAND ${CMAKE_COMMAND} -E env SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} HLS_FILE=${SRC_FILE} HLS_TOP_FUNCTION=${FUNCTION_NAME} HLS_TB_FILE=${TB_FILE} ${VIVADO_HLS} -f ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/synth_hls_function.tcl > hls_${FUNCTION_NAME}.log COMMAND ${CMAKE_COMMAND} -E env HLS_DIR=${CMAKE_CURRENT_BINARY_DIR}/${FUNCTION_NAME}/solution1 CURR_DIR=${CMAKE_CURRENT_BINARY_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/check_hls.sh ${FUNCTION_NAME} COMMAND ${CMAKE_COMMAND} -E copy ${FUNCTION_NAME}/solution1/impl/ip/psi_ch_hls_${FUNCTION_NAME}_1_0.zip . - DEPENDS ${SRC_FILE} hls_jfjoch.h ../include/jfjoch_fpga.h) + DEPENDS ${SRC_FILE} hls_jfjoch.h ../pcie_driver/jfjoch_fpga.h) SET (HLS_IPS ${HLS_IPS} psi_ch_hls_${FUNCTION_NAME}_1_0.zip PARENT_SCOPE) ENDFUNCTION(MAKE_HLS_MODULE) diff --git a/fpga/hls/hls_jfjoch.h b/fpga/hls/hls_jfjoch.h index 2a61fccb..cd0f3294 100644 --- a/fpga/hls/hls_jfjoch.h +++ b/fpga/hls/hls_jfjoch.h @@ -3,7 +3,7 @@ #ifndef JUNGFRAUJOCH_HLS_JFJOCH_H #define JUNGFRAUJOCH_HLS_JFJOCH_H -#include "../include/jfjoch_fpga.h" +#include "../pcie_driver/jfjoch_fpga.h" #include #include @@ -24,7 +24,7 @@ namespace hls { } #endif -#include "../include/jfjoch_fpga.h" +#include "../pcie_driver/jfjoch_fpga.h" typedef ap_ufixed<16,2, AP_RND_CONV> gainG0_t; typedef ap_ufixed<16,4, AP_RND_CONV> gainG1_t; diff --git a/fpga/host_library/CMakeLists.txt b/fpga/host_library/CMakeLists.txt index f1c0a4fd..b4f3e1b7 100644 --- a/fpga/host_library/CMakeLists.txt +++ b/fpga/host_library/CMakeLists.txt @@ -1,4 +1,4 @@ -ADD_LIBRARY(JFJochDevice STATIC JungfraujochDevice.cpp JungfraujochDevice.h ../include/jfjoch_fpga.h) +ADD_LIBRARY(JFJochDevice STATIC JungfraujochDevice.cpp JungfraujochDevice.h ../pcie_driver/jfjoch_fpga.h) TARGET_LINK_LIBRARIES(JFJochDevice JFJochCommon) diff --git a/fpga/host_library/JungfraujochDevice.cpp b/fpga/host_library/JungfraujochDevice.cpp index 84a21a49..7efd86fa 100644 --- a/fpga/host_library/JungfraujochDevice.cpp +++ b/fpga/host_library/JungfraujochDevice.cpp @@ -4,7 +4,7 @@ #include #include -#include "../include/jfjoch_fpga.h" +#include "../pcie_driver/jfjoch_fpga.h" #include "JungfraujochDevice.h" #include "../../common/JFJochException.h" diff --git a/fpga/host_library/JungfraujochDevice.h b/fpga/host_library/JungfraujochDevice.h index abc26380..fcba67e0 100644 --- a/fpga/host_library/JungfraujochDevice.h +++ b/fpga/host_library/JungfraujochDevice.h @@ -4,7 +4,7 @@ #define JUNGFRAUJOCH_JUNGFRAUJOCHDEVICE_H #include -#include "../include/jfjoch_fpga.h" +#include "../pcie_driver/jfjoch_fpga.h" struct JungfraujochDeviceCompletion { uint16_t data_collection_id; diff --git a/fpga/host_library/README.md b/fpga/host_library/README.md index c6d004f1..091dccc5 100644 --- a/fpga/host_library/README.md +++ b/fpga/host_library/README.md @@ -28,7 +28,7 @@ The first step for using the card is configuring network. To use network, one ne The card will receive MAC address automatically based on Xilinx assigned number, but IPv4 address has to be configured with `JungfraujochDevice::SetIPv4Address()` function. The card is equipped with a simple network stack - if both MAC and IPv4 addresses are set and 100G interface is used, the card will periodically send ARP gratuitous messages, it will also reply to ARP requests and to ICMP pings. Given 4x10G interface is designed for direct Jungfraujoch-detector configuration, without a switch, diagnostics functionality is not offered here at the moment. ### Mapping kernel buffers -Next, kernel buffers need to be mapped to the user space. These buffers are allocated with memory physically continuous, simplyfing operation of the card and the driver. Count of these buffers can be checked with `JungfraujochDevice::GetBufferCount()` function. Buffers can be mapped with `JungfraujochDevice::MapKernelBuffer()` function and deallocated with `JungfraujochDevice::UnmapKernelBuffer()` functions. Structure of the kernel buffer is `DeviceOutput` and described in [include/jfjoch_fpga.h](../include/jfjoch_fpga.h) header file. +Next, kernel buffers need to be mapped to the user space. These buffers are allocated with memory physically continuous, simplyfing operation of the card and the driver. Count of these buffers can be checked with `JungfraujochDevice::GetBufferCount()` function. Buffers can be mapped with `JungfraujochDevice::MapKernelBuffer()` function and deallocated with `JungfraujochDevice::UnmapKernelBuffer()` functions. Structure of the kernel buffer is `DeviceOutput` and described in [pcie_driver/jfjoch_fpga.h](../pcie_driver/jfjoch_fpga.h) header file. ### Uploading calibration Uploading calibration goes with two steps: @@ -55,11 +55,11 @@ The following can be uploaded: Before any operation one needs to check if card is idle (not running data collection) with `JungfraujochDevice::IsIdle()` function. Most configuration parameters cannot be changed, when card is in not-idle state. -The card can be then configured with `JungfraujochDevice::SetConfig()` function. Details of the configuration data structure are given in [include/jfjoch_fpga.h](../include/jfjoch_fpga.h) header file. +The card can be then configured with `JungfraujochDevice::SetConfig()` function. Details of the configuration data structure are given in [pcie_driver/jfjoch_fpga.h](../pcie_driver/jfjoch_fpga.h) header file. ### Data collection -Then one can start the card with `JungfraujochDevice::Start()` function. Final step is to wait for first completion (with value `HANDLE_START` defined in [include/jfjoch_fpga.h](../include/jfjoch_fpga.h) as buffer number) using `JungfraujochDevice::ReadWorkCompletion()`. +Then one can start the card with `JungfraujochDevice::Start()` function. Final step is to wait for first completion (with value `HANDLE_START` defined in [pcie_driver/jfjoch_fpga.h](../pcie_driver/jfjoch_fpga.h) as buffer number) using `JungfraujochDevice::ReadWorkCompletion()`. Standard operation of the card requires exchange of buffer ownership between the host application and FPGA card. At the beginning all buffers are owned by host application and should be "given" to the card with `JungfraujochDevice::SendWorkRequest()` function. Then card will wait for the detector to send data. After full module is collected, data are written via Direct Memory Access to host memory and kernel driver is informed with an interrupt that data are ready. Host application can "learn" what was collected by the card by running `JungfraujochDevice::ReadWorkCompletion()` function. Buffer returned by the function is owned by the host application and is safe to process. After processing the buffer has to be given back to card via `JungfraujochDevice::SendWorkRequest()`. If the card doesn't receive enough work requests (open buffers) it won't be able to receive data, resulting in lost packets. @@ -85,7 +85,7 @@ To load the data, one needs to place content of each module (in 16-bit) into res One also needs to switch data source by executing `JungfraujochDevice::SetDataSource()` with respective value. -The next step is to do all the preparations to start data collection, up to `JungfraujochDevice::Start()` and completion handshake. Then one can run `JungfraujochDevice::RunFrameGenerator()` function, with parameters described in the [include/jfjoch_fpga.h](../include/jfjoch_fpga.h) header file. The function is asynchronous, and will start generation, but doesn't wait for the end. Though one can assume that frame generator is done, when data collection is finished. +The next step is to do all the preparations to start data collection, up to `JungfraujochDevice::Start()` and completion handshake. Then one can run `JungfraujochDevice::RunFrameGenerator()` function, with parameters described in the [pcie_driver/jfjoch_fpga.h](../pcie_driver/jfjoch_fpga.h) header file. The function is asynchronous, and will start generation, but doesn't wait for the end. Though one can assume that frame generator is done, when data collection is finished. ### Spot finding parameters Spot finding parameters can be updated with function `JungfraujochDevice::SetSpotFinderParameters()`. diff --git a/fpga/pcie_driver/jfjoch_drv.h b/fpga/pcie_driver/jfjoch_drv.h index fe93f768..ab1611f3 100644 --- a/fpga/pcie_driver/jfjoch_drv.h +++ b/fpga/pcie_driver/jfjoch_drv.h @@ -12,7 +12,7 @@ #include #include -#include "../include/jfjoch_fpga.h" +#include "jfjoch_fpga.h" // From Xilinx XDMA /* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */ diff --git a/fpga/include/jfjoch_fpga.h b/fpga/pcie_driver/jfjoch_fpga.h similarity index 100% rename from fpga/include/jfjoch_fpga.h rename to fpga/pcie_driver/jfjoch_fpga.h diff --git a/fpga/pcie_driver/jfjoch_ioctl.h b/fpga/pcie_driver/jfjoch_ioctl.h index 06f908f9..218b7fbe 100644 --- a/fpga/pcie_driver/jfjoch_ioctl.h +++ b/fpga/pcie_driver/jfjoch_ioctl.h @@ -3,7 +3,7 @@ #ifndef JUNGFRAUJOCH_JFJOCH_IOCTL_H #define JUNGFRAUJOCH_JFJOCH_IOCTL_H -#include "../include/jfjoch_fpga.h" +#include "jfjoch_fpga.h" #ifdef __KERNEL__ #include diff --git a/fpga/pcie_driver/pack.sh b/fpga/pcie_driver/pack.sh new file mode 100755 index 00000000..a66b7053 --- /dev/null +++ b/fpga/pcie_driver/pack.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +GIT_SHA1=`git describe --match=NeVeRmAtCh --always --abbrev=8` +RELEASE_LEVEL=`grep "#define JFJOCH_FPGA_RELEASE" jfjoch_fpga.h | awk -F"0x" '{print $2}'` + +tar cvzf jfjoch_driver_rel${RELEASE_LEVEL}_${GIT_SHA1}.tar.gz *.c *.h README.md Makefile dkms.conf install_dkms.sh >/dev/null 2>&1 diff --git a/fpga/scripts/setup_action.sh b/fpga/scripts/setup_action.sh index bddb2ba6..4f5ffb01 100644 --- a/fpga/scripts/setup_action.sh +++ b/fpga/scripts/setup_action.sh @@ -29,9 +29,9 @@ EOF cp ${SRC_DIR}/hdl/*.v action/hw/hdl # Update action type and release level based on jfjoch_fpga.h -ACTION_TYPE=`grep "#define JFJOCH_FPGA_MAGIC" ${SRC_DIR}/include/jfjoch_fpga.h | awk -F"0x" '{print $2}'` -RELEASE_LEVEL=`grep "#define JFJOCH_FPGA_RELEASE" ${SRC_DIR}/include/jfjoch_fpga.h | awk -F"0x" '{print $2}'` -MAX_MODULES_FPGA=`grep "#define MAX_MODULES_FPGA" ${SRC_DIR}/include/jfjoch_fpga.h |tr -s " " |cut -f3 -d" "` +ACTION_TYPE=`grep "#define JFJOCH_FPGA_MAGIC" ${SRC_DIR}/pcie_driver/jfjoch_fpga.h | awk -F"0x" '{print $2}'` +RELEASE_LEVEL=`grep "#define JFJOCH_FPGA_RELEASE" ${SRC_DIR}/pcie_driver/jfjoch_fpga.h | awk -F"0x" '{print $2}'` +MAX_MODULES_FPGA=`grep "#define MAX_MODULES_FPGA" ${SRC_DIR}/pcie_driver/jfjoch_fpga.h |tr -s " " |cut -f3 -d" "` GIT_SHA1=`git describe --match=NeVeRmAtCh --always --abbrev=8` diff --git a/frame_serialize/CMakeLists.txt b/frame_serialize/CMakeLists.txt index a585cb73..bc75849a 100644 --- a/frame_serialize/CMakeLists.txt +++ b/frame_serialize/CMakeLists.txt @@ -17,6 +17,8 @@ ADD_LIBRARY(ImagePusher STATIC TestImagePusher.cpp TestImagePusher.h ZMQStream2PusherGroup.cpp ZMQStream2PusherGroup.h ZMQStream2Pusher.cpp - ZMQStream2Pusher.h) + ZMQStream2Pusher.h + DumpCBORToFilePusher.cpp + DumpCBORToFilePusher.h) TARGET_LINK_LIBRARIES(ImagePusher CBORStream2FrameSerialize JFJochCommon Compression) \ No newline at end of file diff --git a/frame_serialize/DumpCBORToFilePusher.cpp b/frame_serialize/DumpCBORToFilePusher.cpp new file mode 100644 index 00000000..7c08f86c --- /dev/null +++ b/frame_serialize/DumpCBORToFilePusher.cpp @@ -0,0 +1,59 @@ +// Copyright (2019-2024) Paul Scherrer Institute + +#include "DumpCBORToFilePusher.h" +#include +#include "../include/spdlog/fmt/fmt.h" + +void DumpCBORToFilePusher::StartDataCollection(StartMessage &message) { + std::unique_lock ul(m); + dataset_number++; + img_number = 0; + + size_t approx_size = 1024*1024; + for (const auto &x : message.pixel_mask) + approx_size += x.size; + + std::vector serialization_buffer(approx_size); + CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size()); + serializer.SerializeSequenceStart(message); + + write(fmt::format("dataset{:03d}_start.cbor", dataset_number), + serialization_buffer.data(), serializer.GetBufferSize()); +} + +bool DumpCBORToFilePusher::EndDataCollection(const EndMessage &message) { + std::unique_lock ul(m); + size_t approx_size = 1024*1024; + + std::vector serialization_buffer(approx_size); + CBORStream2Serializer serializer(serialization_buffer.data(), serialization_buffer.size()); + serializer.SerializeSequenceEnd(message); + + write(fmt::format("dataset{:03d}_end.cbor", dataset_number), + serialization_buffer.data(), serializer.GetBufferSize()); + return true; +} + +bool DumpCBORToFilePusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) { + std::unique_lock ul(m); + + write(fmt::format("dataset{:03d}_img{:06d}.cbor", dataset_number, img_number), image_data, image_size); + img_number++; + return true; +} + +void DumpCBORToFilePusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, + ZeroCopyReturnValue *z) { + SendImage(image_data, image_size, image_number); + z->release(); +} + +bool DumpCBORToFilePusher::SendCalibration(const CompressedImage &message) { + return true; +} + +void DumpCBORToFilePusher::write(const std::string &filename, const void *data, size_t data_size) { + std::ofstream file(filename.c_str(), std::ios::out | std::ios::binary | std::ios::trunc); + file.write((char *) data, data_size); + file.close(); +} diff --git a/frame_serialize/DumpCBORToFilePusher.h b/frame_serialize/DumpCBORToFilePusher.h new file mode 100644 index 00000000..f48986f3 --- /dev/null +++ b/frame_serialize/DumpCBORToFilePusher.h @@ -0,0 +1,23 @@ +// Copyright (2019-2024) Paul Scherrer Institute + +#ifndef JUNGFRAUJOCH_DUMPCBORTOFILEPUSHER_H +#define JUNGFRAUJOCH_DUMPCBORTOFILEPUSHER_H + +#include "ImagePusher.h" +#include + +class DumpCBORToFilePusher : public ImagePusher { + std::mutex m; + uint64_t dataset_number = 0; + uint64_t img_number = 0; + static void write(const std::string &filename, const void *data, size_t data_size); +public: + void StartDataCollection(StartMessage &message) override; + bool EndDataCollection(const EndMessage &message) override; + bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override; + void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, ZeroCopyReturnValue *z) override; + bool SendCalibration(const CompressedImage &message) override; +}; + + +#endif //JUNGFRAUJOCH_DUMPCBORTOFILEPUSHER_H diff --git a/frame_serialize/TestImagePusher.cpp b/frame_serialize/TestImagePusher.cpp index 2a7d9332..804fc957 100644 --- a/frame_serialize/TestImagePusher.cpp +++ b/frame_serialize/TestImagePusher.cpp @@ -55,18 +55,7 @@ bool TestImagePusher::SendImage(const uint8_t *image_data, size_t image_size, in void TestImagePusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number, ZeroCopyReturnValue *z) { - std::unique_lock ul(m); - - frame_counter++; - if (image_number == image_id) { - auto deserialized = CBORStream2Deserialize(image_data, image_size); - if (deserialized->data_message) { - receiver_generated_image.resize(deserialized->data_message->image.size); - memcpy(receiver_generated_image.data(), - deserialized->data_message->image.data, - deserialized->data_message->image.size); - } - } + SendImage(image_data, image_size, image_number); z->release(); } diff --git a/receiver/FrameTransformation.h b/receiver/FrameTransformation.h index 453478a4..9f790465 100644 --- a/receiver/FrameTransformation.h +++ b/receiver/FrameTransformation.h @@ -6,7 +6,7 @@ #include "../common/DiffractionExperiment.h" #include "JFJochCompressor.h" #include "../frame_serialize/JFJochMessages.h" -#include "../fpga/include/jfjoch_fpga.h" +#include "../fpga/pcie_driver/jfjoch_fpga.h" class FrameTransformation { const DiffractionExperiment& experiment; diff --git a/receiver/ImageMetadata.h b/receiver/ImageMetadata.h index 3d188ae5..715e375b 100644 --- a/receiver/ImageMetadata.h +++ b/receiver/ImageMetadata.h @@ -4,7 +4,7 @@ #define JUNGFRAUJOCH_IMAGEMETADATA_H #include -#include "../fpga/include/jfjoch_fpga.h" +#include "../fpga/pcie_driver/jfjoch_fpga.h" #include "../frame_serialize/JFJochMessages.h" #include "../common/DiffractionExperiment.h" diff --git a/tests/test_data/jfjoch_broker_test.py b/tests/test_data/jfjoch_broker_test.py new file mode 100644 index 00000000..6b1bcd24 --- /dev/null +++ b/tests/test_data/jfjoch_broker_test.py @@ -0,0 +1,21 @@ +import requests +import h5py +import hdf5plugin + +requests.post("http://localhost:5232/initialize") + +with h5py.File("compression_benchmark.h5", "r") as f: + img = f["/entry/data/data"] + res = requests.put(url="http://localhost:5232/config/internal_generator_image?number=0", data = img[0, :, :].tobytes()) + +start_input = { + "beam_x_pxl":1090, + "beam_y_pxl":1136, + "detector_distance_mm":75, + "photon_energy_keV":12.4, + "sample_name":"lyso", + "unit_cell":{"a":36.9, "b":78.95, "c":78.95, "alpha":90, "beta":90, "gamma":90}, + "images_per_trigger":5, + "file_prefix":"lyso_test" +} +requests.post("http://localhost:5232/start", json=start_input) diff --git a/writer/REAMDE.md b/writer/REAMDE.md index 98d37bd1..5d17685d 100644 --- a/writer/REAMDE.md +++ b/writer/REAMDE.md @@ -17,7 +17,7 @@ jfjoch_writer {options}
Options: -H | --http_port= HTTP port for statistics -r | --zmq_repub_port= ZeroMQ port for PUSH socket to republish images --f | --zmq_file_port= ZeroMQ port for PUB socket to inform about finalized files +-f | --zmq_file_port= ZeroMQ port for PUB socket for notifications on finalized files ``` for example: @@ -35,21 +35,35 @@ Republish creates a PULL socket on the writer, where all the messages are republ Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won't be republished. In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won't be retried. -Republish address is optional, if omitted this functionality is not enabled. +Republish functionality is optional, if republish port number is omitted this functionality is not enabled. ## Finalized files information Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure: ``` { - "filename": : HDF5 data file name, + "filename": : HDF5 data file name (relative to writer root directory), "nimages": number of images in the file, "user_data": or user_data } ``` -`user_data` is defined as `header_appendix` in the `/start` operation in the `jfjoch_broker`. -If the `header_appendix` is a string with valid JSON meaning, it will be transferred as JSON. +`user_data` is defined as `header_appendix` in the `/start` operation in the `jfjoch_broker`. +If the `header_appendix` is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. +For example `header_appendix` of `{"param1": "test1", "param2": ["test1", "test2"]}`, than example message will look as follows: +```json +{ + "filename": "dataset_name_data_000001.h5", + "nimages": 1000, + "user_data": { + "param1": "test1", + "param2": ["test1", "test2"] + } +} +``` + + +Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled. ## NXmx extensions Jungfraujoch aims to generate files compliant with NXmx format, as well as make them as close as possible to files written by DECTRIS Filewriter. This ensures the file compatibility of Neggia and Durin XDS plugins, as well as Albula viewer.