mx/Jungfraujoch
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 37 Wiki Activity

Initial commit

Browse Source 
Signed-off-by: Filip Leonarski <filip.leonarski@psi.ch>
This commit is contained in:
leonarski_f
2023-04-06 11:06:26 +02:00
commit 1757d42182
493 changed files with 170747 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
.gitlab-ci.yml Normal file
View File

@@ -0,0 +1,218 @@
stages:
- build
- test
- synthesis
build:x86:gcc:
stage: build
variables:
GIT_SUBMODULE_STRATEGY: recursive
CC: gcc
CXX: g++
tags:
- gcc
- x86
needs: []
script:
- mkdir build
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release ..
- make -j48 jfjoch
build:x86:icpc:
stage: build
variables:
GIT_SUBMODULE_STRATEGY: recursive
CC: icx
CXX: icpx
tags:
- oneapi
- x86
needs: []
script:
- source /opt/grpc/grpc.sh
- source /opt/intel/oneapi/setvars.sh
- mkdir build
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON ..
- make -j48 jfjoch
build:x86:aocc:
stage: build
variables:
GIT_SUBMODULE_STRATEGY: recursive
CC: clang
CXX: clang++
tags:
- aocc
- x86
needs: []
script:
- source /opt/grpc/grpc.sh
- source /opt/AMD/aocc-compiler-4.0.0/setenv_AOCC.sh
- mkdir build
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON ..
- make -j48 jfjoch
build:x86:vitis_hls:
stage: build
variables:
GIT_SUBMODULE_STRATEGY: recursive
tags:
- x86
needs: []
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
changes:
- receiver/hls/*
- receiver/hdl/*
- receiver/scripts/*
- receiver/xdc/*
- receiver/microblaze/*
- common/Definitions.h
script:
- source /opt/Xilinx/Vitis_HLS/2022.1/settings64.sh
- mkdir build
- cd build
- cmake3 ..
- make hls
test:x86:gcc:
stage: test
timeout: 90m
variables:
GIT_SUBMODULE_STRATEGY: recursive
CTEST_OUTPUT_ON_FAILURE: 1
CC: gcc
CXX: g++
needs: ["build:x86:gcc"]
dependencies: []
tags:
- gcc
- x86
- ib
script:
- mkdir -p build
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release -DJFJOCH_COMPILE_TESTS=ON ..
- make -j48 CatchTest CompressionBenchmark HDF5DatasetWriteTest DataAnalysisPerfTest JFCalibrationPerfTest
- cd tests
- ./CatchTest -r junit -o report.xml
- cd ../tools
- ./HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5
- numactl -m 0 -N 0 ./CompressionBenchmark ../../tests/test_data/compression_benchmark.h5
- numactl -m 0 -N 0 ./DataAnalysisPerfTest ../../tests/test_data/compression_benchmark.h5
- numactl -m 0 -N 0 ./JFCalibrationPerfTest
artifacts:
expire_in: 1 week
reports:
junit: build/tests/report.xml
test:x86:crystfel:
stage: test
timeout: 90m
variables:
GIT_SUBMODULE_STRATEGY: recursive
CTEST_OUTPUT_ON_FAILURE: 1
CC: gcc
CXX: g++
needs: ["build:x86:gcc"]
dependencies: []
tags:
- gcc
- x86
- crystfel
script:
- source /opt/grpc/grpc.sh
- mkdir -p build
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release -DgRPC_FROM_SYSTEM=1 -DJFJOCH_COMPILE_TESTS=ON ..
- make -j8 HDF5DatasetWriteTest
- cd ../tests/crystfel
- HDF5DATASET_WRITE_TEST_IMAGES_PER_FILE=0 ../../build/tools/HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 10
- indexamajig -i writing_test.lst -g jf4m.geom -o x.stream --indexing=xgandalf
test:x86:xds_durin:
stage: test
timeout: 90m
variables:
GIT_SUBMODULE_STRATEGY: recursive
CTEST_OUTPUT_ON_FAILURE: 1
CC: gcc
CXX: g++
needs: ["build:x86:gcc"]
dependencies: []
tags:
- gcc
- x86
- xds
script:
- source /opt/rh/devtoolset-10/enable
- source /opt/grpc/grpc.sh
- mkdir -p build
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release -DgRPC_FROM_SYSTEM=1 -DJFJOCH_COMPILE_TESTS=ON ..
- make -j8 HDF5DatasetWriteTest
- cd ../tests/xds_durin
- HDF5DATASET_WRITE_TEST_IMAGES_PER_FILE=0 ../../build/tools/HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 100
- xds_par |grep -a1 ISa |tail -n1
test:x86:xia2.ssx:
stage: test
timeout: 90m
variables:
GIT_SUBMODULE_STRATEGY: recursive
CTEST_OUTPUT_ON_FAILURE: 1
CC: gcc
CXX: g++
needs: ["build:x86:gcc"]
dependencies: []
tags:
- gcc
- x86
- xds
script:
- source /opt/rh/devtoolset-10/enable
- source /opt/grpc/grpc.sh
- mkdir -p build
- mkdir -p dials_test
- cd build
- cmake3 -DCMAKE_BUILD_TYPE=Release -DgRPC_FROM_SYSTEM=1 -DJFJOCH_COMPILE_TESTS=ON ..
- make -j8 HDF5DatasetWriteTest
- cd ../dials_test
- ../build/tools/HDF5DatasetWriteTest ../tests/test_data/compression_benchmark.h5 100
- source /usr/local/dials-v3-13-0/dials_env.sh
- xia2.ssx image=writing_test_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000
synthesis:vivado_pcie:
stage: synthesis
variables:
GIT_SUBMODULE_STRATEGY: recursive
CC: gcc
CXX: g++
allow_failure: true
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
changes:
- receiver/hls/*
- receiver/hdl/*
- receiver/scripts/*
- receiver/xdc/*
- common/Definitions.h
tags:
- vivado
artifacts:
paths:
- build/receiver/*.mcs
- build/receiver/*.bit
expire_in: 1 week
script:
- source /opt/grpc/grpc.sh
- source /opt/Xilinx/Vivado/2022.1/settings64.sh
- mkdir -p build
- cd build
- cmake3 ..
- make action_pcie
needs: ["build:x86:gcc", "build:x86:vitis_hls", "test:x86:gcc"]

13
.gitmodules vendored Normal file
View File

@@ -0,0 +1,13 @@
[submodule "indexing/fast-feedback-indexer"]
path = indexing/fast-feedback-indexer
url = https://github.com/paulscherrerinstitute/fast-feedback-indexer/
[submodule "frame_serialize/tinycbor"]
path = frame_serialize/tinycbor
url = https://github.com/intel/tinycbor
[submodule "compression/zstd"]
path = compression/zstd
url = https://github.com/facebook/zstd
[submodule "detector_control/slsDetectorPackage"]
path = detector_control/slsDetectorPackage
url = https://github.com/slsdetectorgroup/slsDetectorPackage
branch = "developer"

76
CMakeLists.txt Normal file
View File

@@ -0,0 +1,76 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.18)
PROJECT(Jungfraujoch VERSION 1.0 LANGUAGES C CXX CUDA)
SET(CMAKE_CXX_STANDARD 20)
SET(CMAKE_CXX_STANDARD_REQUIRED True)
SET(CMAKE_CUDA_ARCHITECTURES 70 75 80 86) # V100, T4, A100, RTX A4000
SET(CMAKE_CUDA_STANDARD 17)
SET(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native")
SET(CMAKE_C_FLAGS_RELEASE "-O3 -march=native -mtune=native")
SET(CMAKE_CUDA_FLAGS_RELEASE "-O3")
SET(JFJOCH_COMPILE_WRITER ON CACHE BOOL "Compile HDF5 writer")
SET(JFJOCH_COMPILE_FPGA ON CACHE BOOL "Compile FPGA part")
SET(JFJOCH_COMPILE_DETECTOR ON CACHE BOOL "Compile detector control")
SET(JFJOCH_COMPILE_INDEXER ON CACHE BOOL "Compile indexer")
SET(JFJOCH_COMPILE_TESTS OFF CACHE BOOL "Compile tests")
INCLUDE_DIRECTORIES(include)
INCLUDE(CheckIncludeFile)
FIND_LIBRARY(NUMA_LIBRARY NAMES numa DOC "NUMA Library")
CHECK_INCLUDE_FILE(numaif.h HAS_NUMAIF)
CHECK_INCLUDE_FILE(numa.h HAS_NUMA_H)
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
FIND_PACKAGE(ZeroMQ 4 REQUIRED)
ADD_SUBDIRECTORY(jungfrau)
ADD_SUBDIRECTORY(frame_serialize)
ADD_SUBDIRECTORY(grpc)
ADD_SUBDIRECTORY(compression)
ADD_SUBDIRECTORY(common)
ADD_SUBDIRECTORY(broker)
ADD_SUBDIRECTORY(etc)
ADD_SUBDIRECTORY(indexing)
SET(jfjoch_executables jfjoch_broker)
IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_FPGA)
ADD_SUBDIRECTORY(receiver)
LIST(APPEND jfjoch_executables jfjoch_receiver)
ENDIF()
IF (JFJOCH_COMPILE_DETECTOR)
ADD_SUBDIRECTORY(detector_control)
LIST(APPEND jfjoch_executables jfjoch_detector)
ENDIF()
IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_WRITER)
ADD_SUBDIRECTORY(writer)
ADD_SUBDIRECTORY(tools)
LIST(APPEND jfjoch_executables jfjoch_writer jfjoch_writer_multi)
ENDIF()
IF (JFJOCH_COMPILE_TESTS)
ADD_SUBDIRECTORY(tests)
LIST(APPEND jfjoch_executables CatchTest DataAnalysisPerfTest CompressionBenchmark HDF5DatasetWriteTest)
ENDIF()
LIST(APPEND jfjoch_executables jfjoch_udp_simulator)
ADD_CUSTOM_COMMAND(OUTPUT frontend_ui/build/index.html
COMMAND npm run build
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/frontend_ui
DEPENDS jfjoch-grpc-js)
ADD_CUSTOM_TARGET(frontend DEPENDS frontend_ui/build/index.html)
ADD_CUSTOM_TARGET(jfjoch DEPENDS ${jfjoch_executables})
IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
SET(CMAKE_INSTALL_PREFIX /opt/jfjoch CACHE PATH "Default directory" FORCE)
ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)

674
LICENSE.GPL Normal file
View File

@@ -0,0 +1,674 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.

289
LICENSE.OHL-S Normal file
View File

@@ -0,0 +1,289 @@
CERN Open Hardware Licence Version 2 - Strongly Reciprocal
Preamble
CERN has developed this licence to promote collaboration among
hardware designers and to provide a legal tool which supports the
freedom to use, study, modify, share and distribute hardware designs
and products based on those designs. Version 2 of the CERN Open
Hardware Licence comes in three variants: CERN-OHL-P (permissive); and
two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this
licence, CERN-OHL-S (strongly reciprocal).
The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in
unmodified form only.
Use of this Licence does not imply any endorsement by CERN of any
Licensor or their designs nor does it imply any involvement by CERN in
their development.
1 Definitions
1.1 'Licence' means this CERN-OHL-S.
1.2 'Compatible Licence' means
a) any earlier version of the CERN Open Hardware licence, or
b) any version of the CERN-OHL-S, or
c) any licence which permits You to treat the Source to which
it applies as licensed under CERN-OHL-S provided that on
Conveyance of any such Source, or any associated Product You
treat the Source in question as being licensed under
CERN-OHL-S.
1.3 'Source' means information such as design materials or digital
code which can be applied to Make or test a Product or to
prepare a Product for use, Conveyance or sale, regardless of its
medium or how it is expressed. It may include Notices.
1.4 'Covered Source' means Source that is explicitly made available
under this Licence.
1.5 'Product' means any device, component, work or physical object,
whether in finished or intermediate form, arising from the use,
application or processing of Covered Source.
1.6 'Make' means to create or configure something, whether by
manufacture, assembly, compiling, loading or applying Covered
Source or another Product or otherwise.
1.7 'Available Component' means any part, sub-assembly, library or
code which:
a) is licensed to You as Complete Source under a Compatible
Licence; or
b) is available, at the time a Product or the Source containing
it is first Conveyed, to You and any other prospective
licensees
i) as a physical part with sufficient rights and
information (including any configuration and
programming files and information about its
characteristics and interfaces) to enable it either to
be Made itself, or to be sourced and used to Make the
Product; or
ii) as part of the normal distribution of a tool used to
design or Make the Product.
1.8 'Complete Source' means the set of all Source necessary to Make
a Product, in the preferred form for making modifications,
including necessary installation and interfacing information
both for the Product, and for any included Available Components.
If the format is proprietary, it must also be made available in
a format (if the proprietary tool can create it) which is
viewable with a tool available to potential licensees and
licensed under a licence approved by the Free Software
Foundation or the Open Source Initiative. Complete Source need
not include the Source of any Available Component, provided that
You include in the Complete Source sufficient information to
enable a recipient to Make or source and use the Available
Component to Make the Product.
1.9 'Source Location' means a location where a Licensor has placed
Covered Source, and which that Licensor reasonably believes will
remain easily accessible for at least three years for anyone to
obtain a digital copy.
1.10 'Notice' means copyright, acknowledgement and trademark notices,
Source Location references, modification notices (subsection
3.3(b)) and all notices that refer to this Licence and to the
disclaimer of warranties that are included in the Covered
Source.
1.11 'Licensee' or 'You' means any person exercising rights under
this Licence.
1.12 'Licensor' means a natural or legal person who creates or
modifies Covered Source. A person may be a Licensee and a
Licensor at the same time.
1.13 'Convey' means to communicate to the public or distribute.
2 Applicability
2.1 This Licence governs the use, copying, modification, Conveying
of Covered Source and Products, and the Making of Products. By
exercising any right granted under this Licence, You irrevocably
accept these terms and conditions.
2.2 This Licence is granted by the Licensor directly to You, and
shall apply worldwide and without limitation in time.
2.3 You shall not attempt to restrict by contract or otherwise the
rights granted under this Licence to other Licensees.
2.4 This Licence is not intended to restrict fair use, fair dealing,
or any other similar right.
3 Copying, Modifying and Conveying Covered Source
3.1 You may copy and Convey verbatim copies of Covered Source, in
any medium, provided You retain all Notices.
3.2 You may modify Covered Source, other than Notices, provided that
You irrevocably undertake to make that modified Covered Source
available from a Source Location should You Convey a Product in
circumstances where the recipient does not otherwise receive a
copy of the modified Covered Source. In each case subsection 3.3
shall apply.
You may only delete Notices if they are no longer applicable to
the corresponding Covered Source as modified by You and You may
add additional Notices applicable to Your modifications.
Including Covered Source in a larger work is modifying the
Covered Source, and the larger work becomes modified Covered
Source.
3.3 You may Convey modified Covered Source (with the effect that You
shall also become a Licensor) provided that You:
a) retain Notices as required in subsection 3.2;
b) add a Notice to the modified Covered Source stating that You
have modified it, with the date and brief description of how
You have modified it;
c) add a Source Location Notice for the modified Covered Source
if You Convey in circumstances where the recipient does not
otherwise receive a copy of the modified Covered Source; and
d) license the modified Covered Source under the terms and
conditions of this Licence (or, as set out in subsection
8.3, a later version, if permitted by the licence of the
original Covered Source). Such modified Covered Source must
be licensed as a whole, but excluding Available Components
contained in it, which remain licensed under their own
applicable licences.
4 Making and Conveying Products
You may Make Products, and/or Convey them, provided that You either
provide each recipient with a copy of the Complete Source or ensure
that each recipient is notified of the Source Location of the Complete
Source. That Complete Source is Covered Source, and You must
accordingly satisfy Your obligations set out in subsection 3.3. If
specified in a Notice, the Product must visibly and securely display
the Source Location on it or its packaging or documentation in the
manner specified in that Notice.
5 Research and Development
You may Convey Covered Source, modified Covered Source or Products to
a legal entity carrying out development, testing or quality assurance
work on Your behalf provided that the work is performed on terms which
prevent the entity from both using the Source or Products for its own
internal purposes and Conveying the Source or Products or any
modifications to them to any person other than You. Any modifications
made by the entity shall be deemed to be made by You pursuant to
subsection 3.2.
6 DISCLAIMER AND LIABILITY
6.1 DISCLAIMER OF WARRANTY -- The Covered Source and any Products
are provided 'as is' and any express or implied warranties,
including, but not limited to, implied warranties of
merchantability, of satisfactory quality, non-infringement of
third party rights, and fitness for a particular purpose or use
are disclaimed in respect of any Source or Product to the
maximum extent permitted by law. The Licensor makes no
representation that any Source or Product does not or will not
infringe any patent, copyright, trade secret or other
proprietary right. The entire risk as to the use, quality, and
performance of any Source or Product shall be with You and not
the Licensor. This disclaimer of warranty is an essential part
of this Licence and a condition for the grant of any rights
granted under this Licence.
6.2 EXCLUSION AND LIMITATION OF LIABILITY -- The Licensor shall, to
the maximum extent permitted by law, have no liability for
direct, indirect, special, incidental, consequential, exemplary,
punitive or other damages of any character including, without
limitation, procurement of substitute goods or services, loss of
use, data or profits, or business interruption, however caused
and on any theory of contract, warranty, tort (including
negligence), product liability or otherwise, arising in any way
in relation to the Covered Source, modified Covered Source
and/or the Making or Conveyance of a Product, even if advised of
the possibility of such damages, and You shall hold the
Licensor(s) free and harmless from any liability, costs,
damages, fees and expenses, including claims by third parties,
in relation to such use.
7 Patents
7.1 Subject to the terms and conditions of this Licence, each
Licensor hereby grants to You a perpetual, worldwide,
non-exclusive, no-charge, royalty-free, irrevocable (except as
stated in subsections 7.2 and 8.4) patent licence to Make, have
Made, use, offer to sell, sell, import, and otherwise transfer
the Covered Source and Products, where such licence applies only
to those patent claims licensable by such Licensor that are
necessarily infringed by exercising rights under the Covered
Source as Conveyed by that Licensor.
7.2 If You institute patent litigation against any entity (including
a cross-claim or counterclaim in a lawsuit) alleging that the
Covered Source or a Product constitutes direct or contributory
patent infringement, or You seek any declaration that a patent
licensed to You under this Licence is invalid or unenforceable
then any rights granted to You under this Licence shall
terminate as of the date such process is initiated.
8 General
8.1 If any provisions of this Licence are or subsequently become
invalid or unenforceable for any reason, the remaining
provisions shall remain effective.
8.2 You shall not use any of the name (including acronyms and
abbreviations), image, or logo by which the Licensor or CERN is
known, except where needed to comply with section 3, or where
the use is otherwise allowed by law. Any such permitted use
shall be factual and shall not be made so as to suggest any kind
of endorsement or implication of involvement by the Licensor or
its personnel.
8.3 CERN may publish updated versions and variants of this Licence
which it considers to be in the spirit of this version, but may
differ in detail to address new problems or concerns. New
versions will be published with a unique version number and a
variant identifier specifying the variant. If the Licensor has
specified that a given variant applies to the Covered Source
without specifying a version, You may treat that Covered Source
as being released under any version of the CERN-OHL with that
variant. If no variant is specified, the Covered Source shall be
treated as being released under CERN-OHL-S. The Licensor may
also specify that the Covered Source is subject to a specific
version of the CERN-OHL or any later version in which case You
may apply this or any later version of CERN-OHL with the same
variant identifier published by CERN.
8.4 This Licence shall terminate with immediate effect if You fail
to comply with any of its terms and conditions.
8.5 However, if You cease all breaches of this Licence, then Your
Licence from any Licensor is reinstated unless such Licensor has
terminated this Licence by giving You, while You remain in
breach, a notice specifying the breach and requiring You to cure
it within 30 days, and You have failed to come into compliance
in all material respects by the end of the 30 day period. Should
You repeat the breach after receipt of a cure notice and
subsequent reinstatement, this Licence will terminate
immediately and permanently. Section 6 shall continue to apply
after any termination.
8.6 This Licence shall not be enforceable except by a Licensor
acting as such, and third party beneficiary rights are
specifically excluded.

124
README.md Normal file
View File

@@ -0,0 +1,124 @@
# Jungfraujoch
Application to receive data from the JUNGFRAU detector at Swiss Light Source macromolecular crystallography beamlines.
Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang "Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad., 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268).
## License
Software components are licensed with GNU Public License version 3.
Hardware components are licensed with Strongly-reciprocal CERN Open Hardware Licence version 2.
## Hardware requirements
1. JUNGFRAU detector (optimally 4M with 2 kHz enabled read-out boards)
2. Server system with Xilinx Alveo U55C cards and Nvidia GPUs
3. 100G fiber-optic switch between JUNGFRAU and server
## FPGA bitstream
### Dependencies
To build FPGA image you need:
1. Xilinx Vivado 2022.1 or newer
2. License for Ultrascale+ 100G core - license is available at no cost from Xilinx, but needs to be seperatly requested, see [Xilinx website](https://www.xilinx.com/products/intellectual-property/cmac_usplus.html).
Instructions see [here](receiver/README.md)
## Software
### Dependencies
1. C++17 compiler and C++17 standard library (NOT provided by default RHEL 7 installation, need to install Developer Tools, tested with `devtools-11`)
2. CMake version 3.21 or newer + GNU make tool
3. HDF5 library version 1.10 or newer
4. ZeroMQ library
5. Google Remote Procedure Call (gRPC) - see notes below
6. CUDA compiler version 11 or newer (spot finding, indexing, and radial integration)
7. TIFF library with C++ bindings
8. Mellanox OFED - Infinibands Verbs (ibverbs)
9. NUMA library (optional)
Additional dependencies: SLS Detector Package, tinycbor (Intel) and Zstandard (Facebook) are provided as GIT submodules.
Directly included in the repository:
* JSON parser/writer from N. Lohmann - see [github.com/nlohmann/json](https://github.com/nlohmann/json)
* Catch2 testing library - see [github.com/catchorg/Catch2](https://github.com/catchorg/Catch2)
* Xilinx arbitrary precision arithmetic headers - see [github.com/Xilinx/HLS_arbitrary_Precision_Types](https://github.com/Xilinx/HLS_arbitrary_Precision_Types)
* Bitshuffle filter from K. Masui - see [github.com/kiyo-masui/bitshuffle](https://github.com/kiyo-masui/bitshuffle)
* LZ4 compression by Y.Collet - see [github.com/lz4/lz4](https://github.com/lz4/lz4)
* Spdlog logging library - see [github.com/gabime/spdlog](https://github.com/gabime/spdlog)
For license check LICENSE file in respective directory
### Software components
* `jfjoch_receiver` in `fpga/host` - the main component of Jungfraujoch, used to receive data with FPGA smart network cards, compress images and send them over ZeroMQ
* `jfjoch_broker` in `broker` - gRPC based broker, controlling all Jungfraujoch components
* `jfjoch_writer` in `writer` - HDF5 writer
* `jfjoch_detector` in `detector` - wrapper over detector control
Configuration for the modules is given in configuration files present in `etc` directory.
### Installation of gRPC
By default, gRPC will be cloned and compiled automatically, when compiling Jungfraujoch with `cmake`, however this significantly increases installation time.
Alternative is to compile gRPC beforehand. This is very useful especially, when one expects to compile Jungfraujoch multiple times on the same machine.
It is recommended to install gRPC in own directory (not in system path), as there is no easy way to uninstall it.
To compile gRPC:
```
git clone https://github.com/grpc/grpc
cd grpc
git checkout v1.41.1
git submodule update --init
mkdir build
cd build
cmake -DgRPC_ZLIB_PROVIDER="package" -DCMAKE_INSTALL_PREFIX=/opt/grpc ..
make
sudo make install
```
Currently, handling paths for gRPC and its dependencies gives sometimes troubles and few environmental variables need to be setup before compilation (assuming gRPC installed in /opt/grpc):
```
export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:/opt/grpc/lib/cmake:/opt/grpc/lib64/cmake
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/grpc/lib:/opt/grpc/lib64
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/opt/grpc/lib/pkgconfig:/opt/grpc/lib64/pkgconfig
```
### Compilation
Use the following commands (use `cmake` instead of `cmake3` in non-RHEL systems):
```
git submodule update --init
mkdir build
cd build
cmake3 ..
make jfjoch
```
## Tests
To enable compiling test routines use parameter `-DJFJOCH_COMPILE_TESTS=ON` for `cmake`.
Automated test routine is then accessible as `tests/CatchTest`. There are also benchmark routines:
* `CompressionBenchmark` to measure compression bandwidth (single threaded)
* `HDF5DatasetWriteTest` to measure HDF5 dataset writing speed (single threaded)
* `DataAnalysisPerfTest` to measure data analysis performance (single threaded)
* `PedestalPerfTest` to measure pedestal calculation performance
In addition, tests are executed to verify that datasets written by Jungfraujoch are readable with XDS Durin plugin and CrystFEL. Input files for these programs are placed in `xds_durin` and `crystfel` folders. See `.gitlab-ci.yml` for details.
## Building web user interface
### Dependencies
For web user interface:
1. Node.js
2. Web server, e.g. Apache httpd
3. Web grpc
### Building
To build web interface:
```
cd frontend_ui
npm build
```
To install on RHEL 7 and Apache:
```
cd build
sudo cp -r * /var/www/html/
```

13
broker/CMakeLists.txt Normal file
View File

@@ -0,0 +1,13 @@
ADD_LIBRARY(JFJochBroker STATIC
JFJochStateMachine.cpp JFJochStateMachine.h
JFJochServices.cpp JFJochServices.h
JFJochBroker.cpp JFJochBroker.h)
TARGET_LINK_LIBRARIES(JFJochBroker gRPCClients CommonFunctions)
ADD_EXECUTABLE(jfjoch_broker jfjoch_broker.cpp)
TARGET_LINK_LIBRARIES(jfjoch_broker JFJochBroker)
INSTALL(TARGETS jfjoch_broker RUNTIME)

111
broker/JFJochBroker.cpp Normal file
View File

@@ -0,0 +1,111 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "JFJochBroker.h"
#define GRPC_RUN(x) try { {x;} return grpc::Status::OK; } catch (const std::exception& e) { logger.ErrorException(e); return {grpc::StatusCode::ABORTED, e.what()}; }
JFJochBroker::JFJochBroker(const DiffractionExperiment &experiment) {
state_machine.NotThreadSafe_Experiment() = experiment;
}
grpc::Status JFJochBroker::Start(grpc::ServerContext *context, const JFJochProtoBuf::DatasetSettings *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Start(*request) );
}
grpc::Status JFJochBroker::Stop(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Stop() );
}
grpc::Status JFJochBroker::Pedestal(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Pedestal() );
}
grpc::Status JFJochBroker::Initialize(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Initialize() );
}
grpc::Status JFJochBroker::Cancel(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Cancel() );
}
grpc::Status JFJochBroker::Deactivate(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Deactivate() );
}
grpc::Status JFJochBroker::Trigger(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.Trigger() );
}
grpc::Status JFJochBroker::GetStatus(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::BrokerStatus *response) {
GRPC_RUN( *response = state_machine.GetStatus() );
}
grpc::Status JFJochBroker::GetPlots(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::ReceiverDataProcessingPlots *response) {
GRPC_RUN( *response = state_machine.GetPlots() );
}
grpc::Status JFJochBroker::GetPreview(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::PreviewFrame *response) {
GRPC_RUN( *response = services.GetPreviewFrame() );
}
grpc::Status JFJochBroker::GetPedestalG0(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) {
GRPC_RUN( *response = state_machine.GetPedestalG0() );
}
grpc::Status JFJochBroker::GetPedestalG1(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) {
GRPC_RUN( *response = state_machine.GetPedestalG1() );
}
grpc::Status JFJochBroker::GetPedestalG2(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) {
GRPC_RUN( *response = state_machine.GetPedestalG2() );
}
grpc::Status JFJochBroker::GetMask(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) {
GRPC_RUN( *response = state_machine.GetNeXusMask() );
}
grpc::Status JFJochBroker::GetDetectorSettings(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::DetectorSettings *response) {
GRPC_RUN( *response = state_machine.GetDetectorSettings() );
}
grpc::Status JFJochBroker::PutDetectorSettings(grpc::ServerContext *context,
const JFJochProtoBuf::DetectorSettings *request,
JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.SetDetectorSettings(*request) );
}
grpc::Status JFJochBroker::GetDataProcessingSettings(::grpc::ServerContext *context, const ::JFJochProtoBuf::Empty *request,
::JFJochProtoBuf::DataProcessingSettings *response) {
GRPC_RUN( *response = state_machine.GetDataProcessingSettings() );
}
grpc::Status JFJochBroker::PutDataProcessingSettings(::grpc::ServerContext *context,
const ::JFJochProtoBuf::DataProcessingSettings *request,
::JFJochProtoBuf::Empty *response) {
GRPC_RUN( state_machine.SetDataProcessingSettings(*request) );
}
void JFJochBroker::LoadGainFile(const std::string &filename) {
state_machine.LoadGainFile(filename);
logger.Info("Loaded gain file {}", filename);
}
JFJochServices &JFJochBroker::Services() {
return services;
}

77
broker/JFJochBroker.h Normal file
View File

@@ -0,0 +1,77 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JFJOCHBROKER_H
#define JUNGFRAUJOCH_JFJOCHBROKER_H
#include <jfjoch.grpc.pb.h>
#include "JFJochServices.h"
#include "JFJochStateMachine.h"
class JFJochBroker final : public JFJochProtoBuf::gRPC_JFJochBroker::Service {
Logger logger{"JFJochBroker"};
JFJochServices services {logger};
JFJochStateMachine state_machine {services, logger};
public:
JFJochBroker(const DiffractionExperiment& experiment);
void LoadGainFile(const std::string& filename);
JFJochServices& Services();
grpc::Status Start(grpc::ServerContext *context, const JFJochProtoBuf::DatasetSettings *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Stop(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Pedestal(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Initialize(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Cancel(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Deactivate(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Trigger(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status GetStatus(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::BrokerStatus *response) override;
grpc::Status GetPlots(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::ReceiverDataProcessingPlots *response) override;
grpc::Status GetPreview(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::PreviewFrame *response) override;
grpc::Status GetPedestalG0(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) override;
grpc::Status GetDetectorSettings(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::DetectorSettings *response) override;
grpc::Status PutDetectorSettings(grpc::ServerContext *context, const JFJochProtoBuf::DetectorSettings *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status GetDataProcessingSettings(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::DataProcessingSettings *response) override;
grpc::Status PutDataProcessingSettings(grpc::ServerContext *context, const JFJochProtoBuf::DataProcessingSettings *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status GetPedestalG1(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) override;
grpc::Status GetPedestalG2(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) override;
grpc::Status GetMask(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Image *response) override;
};
#endif //JUNGFRAUJOCH_JFJOCHBROKER_H

173
broker/JFJochServices.cpp Normal file
View File

@@ -0,0 +1,173 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "JFJochServices.h"
#include "../common/JFJochException.h"
uint64_t current_time_ms() {
auto curr_time = std::chrono::system_clock::now();
return std::chrono::duration_cast<std::chrono::milliseconds>(curr_time.time_since_epoch()).count();
}
JFJochServices::JFJochServices(Logger &in_logger) : logger(in_logger) {}
void JFJochServices::Start(const DiffractionExperiment& experiment, const JFCalibration &calibration) {
logger.Info("Measurement start for: {}", experiment.GetFilePrefix());
if ((experiment.GetImageNum() > 0) && (!experiment.GetFilePrefix().empty())) {
logger.Info(" ... writer start");
writer.Start(writer_zmq_addr, 0);
writer_running = true;
} else
writer_running = false;
logger.Info(" ... receiver start");
if (experiment.GetDetectorMode() == DetectorMode::Conversion)
receiver.Start(experiment, &calibration);
else
receiver.Start(experiment, nullptr);
if (!experiment.IsUsingInternalPacketGen()) {
logger.Info(" ... detector start");
detector.Start(experiment);
}
logger.Info(" Done!");
}
void JFJochServices::Off() {
detector.Off();
}
void JFJochServices::On(const DiffractionExperiment &x) {
logger.Info("Detector on");
JFJochProtoBuf::DetectorConfig config = x.DetectorConfig(receiver.GetNetworkConfig());
detector.On(config);
logger.Info(" ... done");
}
JFJochProtoBuf::BrokerFullStatus JFJochServices::Stop(const JFCalibration &calibration) {
JFJochProtoBuf::BrokerFullStatus ret;
std::unique_ptr<JFJochException> exception;
try {
logger.Info("Wait for receiver done");
*ret.mutable_receiver() = receiver.Stop();
logger.Info(" ... Receiver efficiency: {} % Max delay: {} Compression ratio {}x",
static_cast<int>(ret.receiver().efficiency()*100.0),
ret.receiver().max_receive_delay(),
static_cast<int>(std::round(ret.receiver().compressed_ratio())));
if (ret.receiver().efficiency() < 1.0) {
for (int i = 0; i < ret.receiver().device_statistics_size(); i++) {
for (int j = 0; j < ret.receiver().device_statistics(i).packets_received_per_module_size(); j++) {
logger.Info(" ... Device: {} Module: {} Packets received: {}", i, j,
ret.receiver().device_statistics(i).packets_received_per_module(j));
}
}
}
} catch (const JFJochException &e) {
logger.Error(" ... finished with error {}",e.what());
exception = std::make_unique<JFJochException>(e);
}
logger.Info("Receiver finished with success");
if (writer_running) {
logger.Info("Stopping writer");
try {
auto stats = writer.Stop();
logger.Info(" ... finished with success");
for (int i = 0; i < stats.size(); i++) {
*ret.add_writer() = stats[i];
logger.Info("Writer {}: Images = {} Throughput = {:.0f} MB/s Frame rate = {:.0f} Hz",
i, stats[i].nimages(), stats[i].performance_mbs(), stats[i].performance_hz());
}
} catch (JFJochException &e) {
logger.Error(" ... finished with error {}",e.what());
exception = std::make_unique<JFJochException>(e);
}
}
logger.Info("Stopping detector");
try {
detector.Stop();
logger.Info(" ... done");
} catch (JFJochException &e) {
logger.Error(" ... finished with error {}",e.what());
exception = std::make_unique<JFJochException>(e);
}
if (exception)
throw JFJochException(*exception);
return ret;
}
void JFJochServices::Abort() {
// Abort should try to achieve the best outcome possible
// but it OK if things fail (for example lost connection)
try {
receiver.Abort();
} catch (const std::exception &e) {
logger.Error(e.what());
}
}
void JFJochServices::Cancel() {
detector.Stop();
receiver.Cancel();
}
JFJochServices &JFJochServices::Receiver(const std::string &addr) {
receiver.Connect(addr);
logger.Info("Using receiver service with gRPC " + addr);
return *this;
}
JFJochServices &JFJochServices::Writer(const std::string &addr, const std::string &zmq_push_addr) {
writer.AddClient(addr);
writer_zmq_addr.push_back(zmq_push_addr);
logger.Info("Using writer service with gRPC {} listening for images from ZeroMQ {}", addr, zmq_push_addr);
return *this;
}
JFJochServices &JFJochServices::Detector(const std::string &addr) {
detector.Connect(addr);
logger.Info("Using detector service with gRPC {}", addr);
return *this;
}
JFJochProtoBuf::ReceiverStatus JFJochServices::GetReceiverStatus() {
return receiver.GetStatus();
}
JFJochProtoBuf::ReceiverDataProcessingPlots JFJochServices::GetPlots() {
try {
return receiver.GetPlots();
} catch (...) {
JFJochProtoBuf::ReceiverDataProcessingPlots plots;
*plots.mutable_indexing_rate() = JFJochProtoBuf::Plot();
*plots.mutable_bkg_estimate() = JFJochProtoBuf::Plot();
*plots.mutable_spot_count() = JFJochProtoBuf::Plot();
*plots.mutable_radial_int_profile() = JFJochProtoBuf::Plot();
return plots;
}
}
void JFJochServices::SetDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings &settings) {
receiver.SetDataProcessingSettings(settings);
}
JFJochProtoBuf::PreviewFrame JFJochServices::GetPreviewFrame() {
return receiver.GetPreviewFrame();
}
void JFJochServices::Trigger() {
detector.Trigger();
}
size_t JFJochServices::WriterZMQCount() const {
return writer_zmq_addr.size();
}

45
broker/JFJochServices.h Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JFJOCHSERVICES_H
#define JUNGFRAUJOCH_JFJOCHSERVICES_H
#include "../common/DiffractionExperiment.h"
#include "../jungfrau/JFCalibration.h"
#include "../common/Logger.h"
#include "../grpc/JFJochReceiverClient.h"
#include "../grpc/JFJochWriterGroupClient.h"
#include "../grpc/JFJochDetectorClient.h"
class JFJochServices {
JFJochReceiverClient receiver;
JFJochWriterGroupClient writer;
JFJochDetectorClient detector;
Logger &logger;
bool writer_running = false;
std::vector<std::string> writer_zmq_addr;
public:
explicit JFJochServices(Logger &in_logger);
void On(const DiffractionExperiment& experiment);
void Off();
void Start(const DiffractionExperiment& experiment, const JFCalibration &calibration);
JFJochProtoBuf::BrokerFullStatus Stop(const JFCalibration &calibration);
void Abort();
void Cancel();
void Trigger();
JFJochProtoBuf::ReceiverStatus GetReceiverStatus();
JFJochProtoBuf::PreviewFrame GetPreviewFrame();
JFJochProtoBuf::ReceiverDataProcessingPlots GetPlots();
void SetDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings &settings);
JFJochServices& Receiver(const std::string &addr);
JFJochServices& Writer(const std::string &addr, const std::string &zmq_push_addr);
JFJochServices& Detector(const std::string &addr);
size_t WriterZMQCount() const;
};
#endif //JUNGFRAUJOCH_JFJOCHSERVICES_H

516
broker/JFJochStateMachine.cpp Normal file
View File

@@ -0,0 +1,516 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <thread>
#include "JFJochStateMachine.h"
#include "../common/JFJochException.h"
JFJochStateMachine::JFJochStateMachine(JFJochServices &in_services, Logger &in_logger)
: services(in_services), logger(in_logger),
data_processing_settings(DiffractionExperiment::DefaultDataProcessingSettings()) {
}
void JFJochStateMachine::ImportPedestalG0(const JFJochProtoBuf::ReceiverOutput &receiver_output) {
if (receiver_output.pedestal_result_size() != experiment.GetModulesNum() * experiment.GetStorageCellNumber())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in pedestal output");
for (int s = 0; s < experiment.GetStorageCellNumber(); s++) {
for (int module = 0; module < experiment.GetModulesNum(); module++)
calibration->Pedestal(module, 0, s)
= receiver_output.pedestal_result(module + s * experiment.GetModulesNum());
}
SetCalibrationStatistics(calibration->GetModuleStatistics());
}
void JFJochStateMachine::ImportPedestal(const JFJochProtoBuf::ReceiverOutput &receiver_output, size_t gain_level,
size_t storage_cell) {
for (int i = 0; i < receiver_output.pedestal_result_size(); i++)
calibration->Pedestal(i, gain_level, storage_cell) = receiver_output.pedestal_result(i);
SetCalibrationStatistics(calibration->GetModuleStatistics());
}
void JFJochStateMachine::TakePedestalInternalAll(std::unique_lock<std::mutex> &ul) {
calibration = std::make_unique<JFCalibration>(experiment);
if (!gain_calibration.empty()) {
if (gain_calibration.size() != experiment.GetModulesNum())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in gain files number");
for (int i = 0; i < gain_calibration.size(); i++)
calibration->GainCalibration(i) = gain_calibration[i];
}
cancel_sequence = false;
logger.Info("Pedestal sequence started");
try {
TakePedestalInternalG0(ul);
for (int i = 0; i < experiment.GetStorageCellNumber(); i++) {
TakePedestalInternalG1(ul, i);
TakePedestalInternalG2(ul, i);
}
} catch (...) {
logger.Info("Pedestal sequence error");
state = JFJochState::Error;
throw;
}
logger.Info("Pedestal sequence done");
}
void JFJochStateMachine::TakePedestalInternalG0(std::unique_lock<std::mutex> &ul) {
state = JFJochState::Pedestal;
DiffractionExperiment local_experiment(experiment);
local_experiment.Mode(DetectorMode::PedestalG0);
if (!cancel_sequence && (local_experiment.GetPedestalG0Frames() > 0)) {
services.Start(local_experiment, *calibration);
services.Trigger();
ul.unlock();
// Allow to cancel/abort during the pedestal data collection
// Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock
auto pedestal_output = services.Stop(*calibration);
ul.lock();
// SetFullMeasurementOutput(pedestal_output);
ImportPedestalG0(pedestal_output.receiver());
}
state = JFJochState::Idle;
}
void JFJochStateMachine::TakePedestalInternalG1(std::unique_lock<std::mutex> &ul, int32_t storage_cell) {
state = JFJochState::Pedestal;
DiffractionExperiment local_experiment(experiment);
local_experiment.Mode(DetectorMode::PedestalG1);
if (local_experiment.GetStorageCellNumber() == 2)
local_experiment.StorageCellStart((storage_cell + 15) % 16); // one previous
if (!cancel_sequence && (local_experiment.GetPedestalG1Frames() > 0)) {
services.Start(local_experiment, *calibration);
services.Trigger();
ul.unlock();
// Allow to cancel/abort during the pedestal data collection
// Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock
auto pedestal_output = services.Stop(*calibration);
ul.lock();
// SetFullMeasurementOutput(pedestal_output);
ImportPedestal(pedestal_output.receiver(), 1, storage_cell);
}
state = JFJochState::Idle;
}
void JFJochStateMachine::TakePedestalInternalG2(std::unique_lock<std::mutex> &ul, int32_t storage_cell) {
state = JFJochState::Pedestal;
DiffractionExperiment local_experiment(experiment);
local_experiment.Mode(DetectorMode::PedestalG2);
if (local_experiment.GetStorageCellNumber() == 2)
local_experiment.StorageCellStart((storage_cell + 15) % 16); // one previous
if (!cancel_sequence && (local_experiment.GetPedestalG2Frames() > 0)) {
services.Start(local_experiment, *calibration);
services.Trigger();
ul.unlock();
// Allow to cancel/abort during the pedestal data collection
// Must ensure that while state is Pedestal, nothing can take lock for longer time, to avoid deadlock
auto pedestal_output = services.Stop(*calibration);
ul.lock();
// SetFullMeasurementOutput(pedestal_output);
ImportPedestal(pedestal_output.receiver(), 2, storage_cell);
}
state = JFJochState::Idle;
}
void JFJochStateMachine::Initialize() {
std::unique_lock<std::mutex> ul(m);
if ((state == JFJochState::Measuring) || (state == JFJochState::Pedestal))
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot initialize during measurement");
logger.Info("Initialize");
state = JFJochState::Busy;
ClearMeasurementStatistics();
try {
services.On(experiment);
} catch (...) {
state = JFJochState::Error;
throw;
}
TakePedestalInternalAll(ul);
}
void JFJochStateMachine::Pedestal() {
std::unique_lock<std::mutex> ul(m);
if (state != JFJochState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,"Must be idle to take pedestal");
TakePedestalInternalAll(ul);
}
void JFJochStateMachine::Trigger() {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Measuring)
services.Trigger();
}
void JFJochStateMachine::Start(const JFJochProtoBuf::DatasetSettings& settings) {
std::unique_lock<std::mutex> ul(m);
if (state != JFJochState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Must be idle to start measurement");
if (measurement.valid())
measurement.get(); // In case measurement was running - clear thread
ClearMeasurementStatistics();
auto mod_settings = settings;
SetDatasetDefaults(mod_settings);
experiment.LoadDatasetSettings(mod_settings);
cancel_sequence = false;
try {
state = JFJochState::Busy;
services.SetDataProcessingSettings(GetDataProcessingSettings());
services.Start(experiment, *calibration);
state = JFJochState::Measuring;
measurement = std::async(std::launch::async, &JFJochStateMachine::WaitTillMeasurementDone, this);
} catch (...) {
state = JFJochState::Error;
services.Abort();
throw;
}
}
void JFJochStateMachine::SetDatasetDefaults(JFJochProtoBuf::DatasetSettings &settings) {
if (settings.detector_distance_mm() <= 0)
settings.set_detector_distance_mm(100);
if (settings.ntrigger() <= 0)
settings.set_ntrigger(1);
}
void JFJochStateMachine::Stop() {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Pedestal)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Cannot use the function during pedestal collection");
c.wait(ul, [&] { return state != JFJochState::Measuring; });
if (!measurement.valid())
return; // This is for unlikely condition of two parallel stops
else
measurement.get();
switch (state) {
case JFJochState::Inactive:
throw JFJochException(JFJochExceptionCategory::WrongDAQState,"Not initialized");
case JFJochState::Error:
throw JFJochException(JFJochExceptionCategory::WrongDAQState,"Detector in error state");
case JFJochState::Measuring:
case JFJochState::Busy:
case JFJochState::Pedestal:
throw JFJochException(JFJochExceptionCategory::WrongDAQState,"Detector in not expected state to end measurment");
case JFJochState::Idle:
break;
}
}
void JFJochStateMachine::WaitTillMeasurementDone() {
try {
auto tmp_output = services.Stop(*calibration);
SetFullMeasurementOutput(tmp_output);
{
std::unique_lock<std::mutex> ul(m);
state = JFJochState::Idle;
}
} catch (...) {
std::unique_lock<std::mutex> ul(m);
state = JFJochState::Error;
}
c.notify_all();
}
void JFJochStateMachine::Abort() {
// This is inconsistency in naming - need to solve later
std::unique_lock<std::mutex> ul(m);
if ((state == JFJochState::Pedestal) || (state == JFJochState::Measuring)) {
services.Abort();
cancel_sequence = true;
}
}
void JFJochStateMachine::Cancel() {
// This is inconsistency in naming - need to solve later
std::unique_lock<std::mutex> ul(m);
if ((state == JFJochState::Pedestal) || (state == JFJochState::Measuring)) {
services.Cancel();
cancel_sequence = true;
}
}
void JFJochStateMachine::DebugOnly_SetState(JFJochState in_state) {
std::unique_lock<std::mutex> ul(m);
state = in_state;
}
void JFJochStateMachine::Deactivate() {
std::unique_lock<std::mutex> ul(m);
try {
if (measurement.valid())
measurement.get();
services.Off();
state = JFJochState::Inactive;
} catch (...) {
state = JFJochState::Error;
throw;
}
}
JFJochStateMachine::~JFJochStateMachine() {
try {
if (measurement.valid())
measurement.get();
} catch (...) {}
}
JFJochProtoBuf::BrokerFullStatus JFJochStateMachine::GetFullMeasurementOutput() const {
std::unique_lock<std::mutex> ul(last_receiver_output_mutex);
return last_receiver_output;
}
void JFJochStateMachine::SetFullMeasurementOutput(JFJochProtoBuf::BrokerFullStatus &output) {
std::unique_lock<std::mutex> ul(last_receiver_output_mutex);
last_receiver_output = output;
auto tmp = JFJochProtoBuf::MeasurementStatistics(); // reset last measurement statistics
tmp.set_file_prefix(experiment.GetFilePrefix());
tmp.set_detector_width(experiment.GetXPixelsNum());
tmp.set_detector_height(experiment.GetYPixelsNum());
tmp.set_detector_pixel_depth(experiment.GetPixelDepth());
if (last_receiver_output.has_receiver()) {
tmp.set_compression_ratio(output.receiver().compressed_ratio());
tmp.set_collection_efficiency(output.receiver().efficiency());
tmp.set_images_collected(output.receiver().images_sent());
tmp.set_cancelled(output.receiver().cancelled());
tmp.set_max_image_number_sent(output.receiver().max_image_number_sent());
tmp.set_max_receive_delay(output.receiver().max_receive_delay());
if (output.receiver().has_indexing_rate())
tmp.set_indexing_rate(output.receiver().indexing_rate());
}
if (last_receiver_output.writer_size() > 0) {
double writer_perf = 0.0;
int64_t images_written = 0;
for (const auto &i: output.writer()) {
writer_perf += i.performance_mbs();
images_written += i.nimages();
}
tmp.set_writer_performance_mbs(writer_perf);
tmp.set_images_written(images_written);
}
measurement_statistics = tmp;
}
void JFJochStateMachine::ClearMeasurementStatistics() {
std::unique_lock<std::mutex> ul(last_receiver_output_mutex);
measurement_statistics.reset();
}
std::optional<JFJochProtoBuf::MeasurementStatistics> JFJochStateMachine::GetMeasurementStatistics() const {
std::unique_lock<std::mutex> ul(last_receiver_output_mutex);
return measurement_statistics;
}
JFCalibration JFJochStateMachine::GetCalibration() const {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Inactive)
throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Detector not calibrated");
return *calibration;
}
void JFJochStateMachine::LoadMask(const std::vector<uint32_t> &vec, uint32_t bit) {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Inactive)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Detector not calibrated");
if (state != JFJochState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot load mask if detector is not idle");
calibration->LoadMask(experiment, vec, bit);
}
JFJochProtoBuf::JFCalibrationStatistics JFJochStateMachine::GetCalibrationStatistics() const {
std::unique_lock<std::mutex> ul(calibration_statistics_mutex);
return calibration_statistics;
}
void JFJochStateMachine::SetCalibrationStatistics(const JFJochProtoBuf::JFCalibrationStatistics &input) {
std::unique_lock<std::mutex> ul(calibration_statistics_mutex);
calibration_statistics = input;
}
JFJochProtoBuf::DetectorSettings JFJochStateMachine::GetDetectorSettings() const {
std::unique_lock<std::mutex> ul(m);
return experiment.GetDetectorSettings();
}
void JFJochStateMachine::SetDetectorSettings(const JFJochProtoBuf::DetectorSettings &settings) {
std::unique_lock<std::mutex> ul(m);
switch (state) {
case JFJochState::Inactive:
case JFJochState::Error:
experiment.LoadDetectorSettings(settings);
break;
case JFJochState::Idle:
experiment.LoadDetectorSettings(settings);
TakePedestalInternalAll(ul);
break;
case JFJochState::Measuring:
case JFJochState::Busy:
case JFJochState::Pedestal:
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot change detector set during data collection");
}
}
DiffractionExperiment &JFJochStateMachine::NotThreadSafe_Experiment() {
return experiment;
}
JFJochProtoBuf::Image JFJochStateMachine::GetNeXusMask() const {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Inactive)
throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Detector not calibrated");
JFJochProtoBuf::Image ret;
auto mask = calibration->CalculateNexusMask(experiment);
ret.set_width(experiment.GetXPixelsNum());
ret.set_height(experiment.GetYPixelsNum());
ret.set_pixel_depth(4);
*ret.mutable_data() = {mask.begin(), mask.end()};
return ret;
}
JFJochProtoBuf::Image JFJochStateMachine::GetPedestalG0() const {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Inactive)
throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Detector not calibrated");
JFJochProtoBuf::Image ret;
auto pedestal = calibration->GetPedestal(0);
ret.set_width(experiment.GetXPixelsNum());
ret.set_height(experiment.GetYPixelsNum());
ret.set_pixel_depth(2);
*ret.mutable_data() = {pedestal.begin(), pedestal.end()};
return ret;
}
JFJochProtoBuf::Image JFJochStateMachine::GetPedestalG1() const {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Inactive)
throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Detector not calibrated");
JFJochProtoBuf::Image ret;
auto pedestal = calibration->GetPedestal(1);
ret.set_width(experiment.GetXPixelsNum());
ret.set_height(experiment.GetYPixelsNum());
ret.set_pixel_depth(2);
*ret.mutable_data() = {pedestal.begin(), pedestal.end()};
return ret;
}
JFJochProtoBuf::Image JFJochStateMachine::GetPedestalG2() const {
std::unique_lock<std::mutex> ul(m);
if (state == JFJochState::Inactive)
throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Detector not calibrated");
JFJochProtoBuf::Image ret;
auto pedestal = calibration->GetPedestal(2);
ret.set_width(experiment.GetXPixelsNum());
ret.set_height(experiment.GetYPixelsNum());
ret.set_pixel_depth(2);
*ret.mutable_data() = {pedestal.begin(), pedestal.end()};
return ret;
}
JFJochProtoBuf::BrokerStatus JFJochStateMachine::GetStatus() const {
JFJochProtoBuf::BrokerStatus ret;
switch (state) {
case JFJochState::Inactive:
ret.set_broker_state(JFJochProtoBuf::NOT_INITIALIZED);
break;
case JFJochState::Idle:
ret.set_broker_state(JFJochProtoBuf::IDLE);
break;
case JFJochState::Measuring:
ret.set_broker_state(JFJochProtoBuf::DATA_COLLECTION);
break;
case JFJochState::Error:
ret.set_broker_state(JFJochProtoBuf::ERROR);
break;
case JFJochState::Busy:
ret.set_broker_state(JFJochProtoBuf::BUSY);
break;
case JFJochState::Pedestal:
ret.set_broker_state(JFJochProtoBuf::PEDESTAL);
break;
}
try {
auto rcv_status = services.GetReceiverStatus();
if (rcv_status.has_progress())
ret.set_progress(rcv_status.progress());
if (rcv_status.has_indexing_rate())
ret.set_indexing_rate(rcv_status.indexing_rate());
} catch (JFJochException &e) {} // ignore exception in getting receiver status (don't really care, e.g. if receiver is down)
return ret;
}
JFJochProtoBuf::ReceiverDataProcessingPlots JFJochStateMachine::GetPlots() const {
return services.GetPlots();
}
void JFJochStateMachine::SetDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings &settings) {
std::unique_lock<std::mutex> ul(data_processing_settings_mutex);
DiffractionExperiment::CheckDataProcessingSettings(settings);
data_processing_settings = settings;
services.SetDataProcessingSettings(data_processing_settings);
}
JFJochProtoBuf::DataProcessingSettings JFJochStateMachine::GetDataProcessingSettings() const {
std::unique_lock<std::mutex> ul(data_processing_settings_mutex);
return data_processing_settings;
}
JFJochState JFJochStateMachine::GetState() const {
return state;
}
void JFJochStateMachine::LoadGainFile(const std::string &filename) {
gain_calibration.emplace_back(filename);
}

106
broker/JFJochStateMachine.h Normal file
View File

@@ -0,0 +1,106 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JFJOCHSTATEMACHINE_H
#define JUNGFRAUJOCH_JFJOCHSTATEMACHINE_H
#include <string>
#include <mutex>
#include <future>
#include <optional>
#include "../common/DiffractionExperiment.h"
#include "../jungfrau/JFCalibration.h"
#include "../common/Logger.h"
#include "JFJochServices.h"
enum class JFJochState {Inactive, Idle, Measuring, Error, Busy, Pedestal};
class JFJochStateMachine {
Logger &logger;
JFJochServices &services;
mutable std::mutex m;
std::condition_variable c;
// mutex m is protecting:
DiffractionExperiment experiment;
volatile JFJochState state = JFJochState::Inactive;
volatile bool cancel_sequence = false;
std::unique_ptr<JFCalibration> calibration;
std::vector<JFModuleGainCalibration> gain_calibration;
std::future<void> measurement;
mutable std::mutex calibration_statistics_mutex;
JFJochProtoBuf::JFCalibrationStatistics calibration_statistics;
mutable std::mutex last_receiver_output_mutex;
JFJochProtoBuf::BrokerFullStatus last_receiver_output;
std::optional<JFJochProtoBuf::MeasurementStatistics> measurement_statistics;
void SetFullMeasurementOutput(JFJochProtoBuf::BrokerFullStatus &output);
void ClearMeasurementStatistics();
JFJochProtoBuf::BrokerFullStatus GetFullMeasurementOutput() const;
mutable std::mutex data_processing_settings_mutex;
JFJochProtoBuf::DataProcessingSettings data_processing_settings;
// Private functions assume that lock m is acquired
void SetDatasetDefaults(JFJochProtoBuf::DatasetSettings& settings);
void WaitTillMeasurementDone();
void ImportPedestal(const JFJochProtoBuf::ReceiverOutput &receiver_output, size_t gain_level, size_t storage_cell = 0);
void ImportPedestalG0(const JFJochProtoBuf::ReceiverOutput &receiver_output);
void TakePedestalInternalAll(std::unique_lock<std::mutex> &ul);
void TakePedestalInternalG0(std::unique_lock<std::mutex> &ul);
void TakePedestalInternalG1(std::unique_lock<std::mutex> &ul, int32_t storage_cell = 0);
void TakePedestalInternalG2(std::unique_lock<std::mutex> &ul, int32_t storage_cell = 0);
public:
JFJochStateMachine(JFJochServices &in_services, Logger &logger);
~JFJochStateMachine();
void Initialize();
void Pedestal();
void Deactivate();
void Start(const JFJochProtoBuf::DatasetSettings& settings);
void Stop();
void Trigger();
void Abort();
void Cancel();
void LoadMask(const std::vector<uint32_t> &vec, uint32_t bit);
void SetCalibrationStatistics(const JFJochProtoBuf::JFCalibrationStatistics &input);
JFCalibration GetCalibration() const;
JFJochProtoBuf::DetectorSettings GetDetectorSettings() const;
void SetDetectorSettings(const JFJochProtoBuf::DetectorSettings& settings);
// return by value to ensure thread safety
std::optional<JFJochProtoBuf::MeasurementStatistics> GetMeasurementStatistics() const;
JFJochProtoBuf::JFCalibrationStatistics GetCalibrationStatistics() const;
JFJochProtoBuf::BrokerStatus GetStatus() const;
JFJochProtoBuf::ReceiverDataProcessingPlots GetPlots() const;
JFJochProtoBuf::Image GetNeXusMask() const;
JFJochProtoBuf::Image GetPedestalG0() const;
JFJochProtoBuf::Image GetPedestalG1() const;
JFJochProtoBuf::Image GetPedestalG2() const;
void SetDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings& settings);
JFJochProtoBuf::DataProcessingSettings GetDataProcessingSettings() const;
JFJochState GetState() const;
// Not thread safe - only for configuration in serial context
DiffractionExperiment& NotThreadSafe_Experiment();
// Function for debug only - UNSAFE for real operation
void DebugOnly_SetState(JFJochState state);
void LoadGainFile(const std::string& filename);
};
#endif //JUNGFRAUJOCH_JFJOCHSTATEMACHINE_H

100
broker/jfjoch_broker.cpp Normal file
View File

@@ -0,0 +1,100 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <fstream>
#include <random>
#include <nlohmann/json.hpp>
#include "../common/Logger.h"
#include "../common/NetworkAddressConvert.h"
#include "JFJochBroker.h"
#include "../grpc/gRPCServer_Template.h"
int main (int argc, char **argv) {
if (argc > 3) {
std::cout << "Usage ./jfjoch_broker {<JSON config> {<TCP gRPC port>}}" << std::endl;
exit(EXIT_FAILURE);
}
uint16_t grpc_port = 5232;
if (argc >= 3) grpc_port = atoi(argv[2]);
Logger logger("jfjoch_broker");
nlohmann::json input;
if (argc > 1) {
std::ifstream file(argv[1]);
try {
input = nlohmann::json::parse(file);
} catch (const nlohmann::json::exception &e) {
logger.Error("JSON Parsing exception: " + std::string(e.what()));
exit(EXIT_FAILURE);
}
}
DiffractionExperiment experiment(2, {4,4}, 8, 36);
experiment.PedestalG0Frames(2000).PedestalG1Frames(300).PedestalG2Frames(300);
experiment.MaskChipEdges(true).MaskModuleEdges(true);
try {
experiment.SourceName(input.at("source_name").get<std::string>());
experiment.SourceNameShort(input.at("source_name_short").get<std::string>());
} catch (std::exception &e) {
logger.Warning("Source name metadata error: {}", e.what() );
}
try {
experiment.InstrumentName(input.at("instrument_name").get<std::string>());
experiment.InstrumentNameShort(input.at("instrument_name_short").get<std::string>());
} catch (std::exception &e) {
logger.Warning("Instrument name metadata error: {}", e.what() );
}
std::random_device generator;
uint8_t base_ipv4_addr_net = UINT8_MAX;
if (input.contains("ipv4_subnet"))
base_ipv4_addr_net = input["ipv4_subnet"];
std::uniform_int_distribution<uint8_t> ipv4_subnet_distribution(0,UINT8_MAX-1);
if (base_ipv4_addr_net == UINT8_MAX)
base_ipv4_addr_net = ipv4_subnet_distribution(generator);
experiment.IPv4Subnet("10.1." + std::to_string(base_ipv4_addr_net) + ".0");
logger.Info("Base IPv4 address for FPGA and detector modules: "
+ IPv4AddressToStr(experiment.GetDestIPv4Address(0)));
JFJochBroker broker(experiment);
if (input.contains("receiver_addr"))
broker.Services().Receiver(input["receiver_addr"]);
if (input.contains("writer")) {
if (input["writer"].is_array()) {
for (const auto &j: input["writer"])
broker.Services().Writer(j["addr_grpc"], j["addr_zmq"]);
} else {
broker.Services().Writer(input["writer"]["addr_grpc"], input["writer"]["addr_zmq"]);
}
}
if (input.contains("detector_addr"))
broker.Services().Detector(input["detector_addr"]);
if (input.contains("gain_file") && (input["gain_file"].is_array())) {
for (int i = 0; i < input["gain_file"].size(); i++) {
try {
broker.LoadGainFile(input["gain_file"][i].get<std::string>());
} catch (const std::exception &e) {
logger.ErrorException(e);
exit(EXIT_FAILURE);
}
}
}
std::string grpc_addr = "0.0.0.0:" + std::to_string(grpc_port);
auto server = gRPCServer(grpc_addr, broker);
logger.Info("gRPC configuration listening on address " + grpc_addr);
server->Wait();
}

27
cmake/FindZeroMQ.cmake Normal file
View File

@@ -0,0 +1,27 @@
#From: https://github.com/zeromq/cppzmq/
set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
find_package(PkgConfig)
pkg_check_modules(PC_LIBZMQ QUIET libzmq)
set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION})
find_library(ZeroMQ_LIBRARY NAMES libzmq.so libzmq.dylib libzmq.dll
PATHS ${PC_LIBZMQ_LIBDIR} ${PC_LIBZMQ_LIBRARY_DIRS})
find_library(ZeroMQ_STATIC_LIBRARY NAMES libzmq-static.a libzmq.a libzmq.dll.a
PATHS ${PC_LIBZMQ_LIBDIR} ${PC_LIBZMQ_LIBRARY_DIRS})
if(ZeroMQ_LIBRARY OR ZeroMQ_STATIC_LIBRARY)
set(ZeroMQ_FOUND ON)
endif()
if (TARGET libzmq)
# avoid errors defining targets twice
return()
endif()
add_library(libzmq SHARED IMPORTED)
set_property(TARGET libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PC_LIBZMQ_INCLUDE_DIRS})
set_property(TARGET libzmq PROPERTY IMPORTED_LOCATION ${ZeroMQ_LIBRARY})
add_library(libzmq-static STATIC IMPORTED ${PC_LIBZMQ_INCLUDE_DIRS})
set_property(TARGET libzmq-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PC_LIBZMQ_INCLUDE_DIRS})
set_property(TARGET libzmq-static PROPERTY IMPORTED_LOCATION ${ZeroMQ_STATIC_LIBRARY})

63
common/CMakeLists.txt Normal file
View File

@@ -0,0 +1,63 @@
# git header
# the commit's SHA1, and whether the building workspace was dirty or not
FIND_PACKAGE(Git)
EXECUTE_PROCESS(COMMAND
"${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_SHA1
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
# the date of the commit
EXECUTE_PROCESS(COMMAND
"${GIT_EXECUTABLE}" log -1 --format=%ad --date=local
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_DATE
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
MESSAGE(STATUS "Git SHA1: ${GIT_SHA1}")
MESSAGE(STATUS "Git date: ${GIT_DATE}")
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/GitInfo.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp" @ONLY)
ADD_LIBRARY( CommonFunctions STATIC
Logger.cpp Logger.h
Coord.cpp Coord.h
DiffractionExperiment.cpp DiffractionExperiment.h
RawToConvertedGeometry.h
JFJochException.h
Definitions.h
${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp GitInfo.h
FrameTransformation.cpp FrameTransformation.h
ZMQWrappers.cpp ZMQWrappers.h
ThreadSafeFIFO.h
ZMQPreviewPublisher.cpp ZMQPreviewPublisher.h
ZMQImagePusher.cpp ZMQImagePusher.h
RadialIntegration.cpp RadialIntegration.h
DiffractionSpot.cpp DiffractionSpot.h
StrongPixelSet.cpp StrongPixelSet.h
Latch.cpp Latch.h
RadialIntegrationMapping.cpp RadialIntegrationMapping.h
StatusVector.h
ImagePusher.cpp ImagePusher.h
TestImagePusher.cpp TestImagePusher.h
SpotToSave.h
NetworkAddressConvert.h NetworkAddressConvert.cpp
WriteTIFF.cpp WriteTIFF.h
grpcToJson.h jsonToGrpc.h to_fixed.h
GPUImageAnalysis.h GPUImageAnalysis.cu
DiffractionExperiment.h DiffractionGeometry.cpp)
FIND_LIBRARY(CUDART_LIBRARY cudart_static PATHS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} REQUIRED)
TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration JFJochProtoBuf ${CUDART_LIBRARY} -lrt)
IF(HAS_NUMAIF AND NUMA_LIBRARY)
TARGET_COMPILE_DEFINITIONS(CommonFunctions PRIVATE -DJFJOCH_USE_NUMA)
TARGET_LINK_LIBRARIES(CommonFunctions ${NUMA_LIBRARY})
ENDIF()
FIND_PACKAGE(TIFF REQUIRED)
FIND_LIBRARY(TIFFXX NAMES tiffxx REQUIRED DOC "Tiff C++ library")
TARGET_LINK_LIBRARIES(CommonFunctions TIFF::TIFF ${TIFFXX})

139
common/Coord.cpp Normal file
View File

@@ -0,0 +1,139 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <cmath>
#include "Coord.h"
#include "JFJochException.h"
Coord::Coord() {
x = 0.0; y = 0.0; z = 0.0;
}
Coord::Coord(const float in[3]) {
x = in[0];
y = in[1];
z = in[2];
}
Coord::Coord(float in_x, float in_y, float in_z) {
x = in_x;
y = in_y;
z = in_z;
}
Coord Coord::operator+(const Coord &in) const {
return Coord(this->x+in.x, this->y+in.y, this->z+in.z);
}
Coord Coord::operator-(const Coord &in) const {
return Coord(this->x-in.x, this->y-in.y, this->z-in.z);
}
Coord Coord::operator*(float in) const {
return Coord(this->x*in, this->y*in, this->z*in);
}
Coord Coord::operator/(float in) const {
return Coord(this->x/in, this->y/in, this->z/in);
};
Coord Coord::operator-() const {
return Coord(- this->x, -this->y, -this->z);
}
Coord& Coord::operator+=(const Coord &in) {
this->x += in.x;
this->y += in.y;
this->z += in.z;
return *this;
}
Coord& Coord::operator-=(const Coord &in) {
this->x -= in.x;
this->y -= in.y;
this->z -= in.z;
return *this;
}
Coord& Coord::operator*=(float in) {
this->x *= in;
this->y *= in;
this->z *= in;
return *this;
}
Coord& Coord::operator/=(float in) {
this->x /= in;
this->y /= in;
this->z /= in;
return *this;
}
Coord Coord::operator%(const Coord &in) const {
return Coord(this->y * in.z - this->z * in.y,
this->z * in.x - this->x * in.z,
this->x * in.y - this->y * in.x);
}; // Cross product
float Coord::operator*(const Coord &in) const {
return this->x * in.x + this->y * in.y + this->z * in.z;
};
bool Coord::operator==(const Coord &other) const {
if ((this->x == other.x) && (this->y == other.y) && (this->z == other.z))
return true;
else
return false;
}
float Coord::Length() const {
return sqrt(this->x*this->x + this->y*this->y + this->z*this->z);
}
Coord Coord::Normalize() const {
float len = Length();
return Coord(this->x/len, this->y/len, this->z/len);
}
Coord operator*(float in1, const Coord& in2) {
return in2 * in1;
}
const float& Coord::operator[](int64_t val) const {
switch (val) {
case 0:
return x;
case 1:
return y;
case 2:
return z;
default:
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Coord index must be in range 0-2");
}
}
float& Coord::operator[](int64_t val) {
switch (val) {
case 0:
return x;
case 1:
return y;
case 2:
return z;
default:
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Coord index must be in range 0-2");
}
}
std::ostream &operator<<( std::ostream &output, const Coord &in ) {
output << in.x << " " << in.y << " " << in.z;
return output;
}
float angle_deg(const Coord &c1, const Coord &c2) {
float cos_ang = c1 * c2 / (c1.Length() * c2.Length());
return acos(cos_ang) * (180.0 / M_PI);
}

46
common/Coord.h Normal file
View File

@@ -0,0 +1,46 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef INDEX_COORD_H
#define INDEX_COORD_H
#include <ostream>
class Coord {
public:
float x,y,z;
Coord();
Coord(const float in[3]);
Coord(float x, float y, float z);
Coord operator+(const Coord &in) const;
Coord operator-(const Coord &in) const;
Coord operator*(float in) const;
Coord operator/(float in) const;
Coord operator-() const;
Coord& operator+=(const Coord &in);
Coord& operator-=(const Coord &in);
Coord& operator*=(float in);
Coord& operator/=(float in);
bool operator==(const Coord &other) const;
Coord operator%(const Coord &in) const; // Cross product
float operator*(const Coord &in) const; // Dot product
const float& operator[](int64_t val) const;
float& operator[](int64_t val);
float Length() const;
Coord Normalize() const;
friend std::ostream &operator<<( std::ostream &output, const Coord &in );
};
Coord operator*(float in1, const Coord& in2);
float angle_deg(const Coord &in1, const Coord &in2);
#endif //INDEX_COORD_H

120
common/Definitions.h Normal file
View File

@@ -0,0 +1,120 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef DEFINITIONS_H
#define DEFINITIONS_H
#define WVL_1A_IN_KEV 12.39854f
#define DELAY_FRAMES_STOP_AND_QUIT 5
#define RAW_MODULE_LINES (512L)
#define RAW_MODULE_COLS (1024L)
#define RAW_MODULE_SIZE (RAW_MODULE_LINES * RAW_MODULE_COLS)
#define CONVERTED_MODULE_LINES (514L)
#define CONVERTED_MODULE_COLS (1030L)
#define CONVERTED_MODULE_SIZE (CONVERTED_MODULE_LINES * CONVERTED_MODULE_COLS)
#define JUNGFRAU_PACKET_SIZE_BYTES (8192)
#define FPGA_BUFFER_LOCATION_SIZE (RAW_MODULE_SIZE * sizeof(short))
#define MIN_COUNT_TIME_IN_US 10
#define MIN_FRAME_TIME_HALF_SPEED_IN_US 1000
#define MIN_FRAME_TIME_FULL_SPEED_IN_US 470
#define MAX_FRAME_TIME 2000
#define MAX_SUMMATION 1000
#define READOUT_TIME_IN_US 20
#define GRPC_MAX_MESSAGE_SIZE (1000L*1000L*1000L)
#define MIN_ENERGY 0.1
#define MAX_ENERGY 25.0
#define PEDESTAL_WINDOW_SIZE 128
#define PEDESTAL_WRONG 16384
#define FRAME_TIME_PEDE_G1G2_IN_US (10*1000)
#define SENSOR_THICKNESS_IN_UM 320.0
#define PIXEL_SIZE_IN_UM 75.0
#define PIXEL_SIZE_IN_MM (PIXEL_SIZE_IN_UM/1000.0)
#define SENSOR_MATERIAL "Si"
#define GAIN_G0_MULTIPLIER 32
#define GAIN_G1_MULTIPLIER (-1)
#define GAIN_G2_MULTIPLIER (-1)
#define DEFAULT_G0_FACTOR (41.0)
#define DEFAULT_G1_FACTOR (-1.439)
#define DEFAULT_G2_FACTOR (-0.1145)
// For FPGA
/* This number is unique and is declared in ~snap/ActionTypes.md */
#define ACTION_TYPE 0x52324158
#define RELEASE_LEVEL 0x0033
#define MODE_CONV 0x0001L
#define MODE_INTERNAL_PACKET_GEN 0x0002L
#define TASK_NO_DATA_STREAM UINT16_MAX
#define PIXEL_OUT_SATURATION (INT16_MAX)
#define PIXEL_OUT_LOST (INT16_MIN)
#define PIXEL_OUT_0xFFFF (INT16_MIN)
#define PIXEL_OUT_G1_SATURATION (INT16_MIN)
#define PIXEL_OUT_GAINBIT_2 (INT16_MIN)
#define LOAD_CALIBRATION_BRAM_SIZE 1024
// FPGA register map
#define ADDR_CTRL_REGISTER 0x0000
#define ADDR_GIT_SHA1 0x000C
#define ADDR_ACTION_TYPE 0x0010
#define ADDR_RELEASE_LEVEL 0x0014
#define ADDR_HBM_TEMP 0x0018
#define ADDR_HBM_MAX_TEMP 0x001C
#define ADDR_MAX_MODULES_FPGA 0x0020
#define ADDR_MODS_INT_PKT_GEN 0x0024
#define ADDR_STALLS_HOST_LO 0x0028
#define ADDR_STALLS_HOST_HI 0x002C
#define ADDR_STALLS_HBM_LO 0x0030
#define ADDR_STALLS_HBM_HI 0x0034
#define ADDR_FIFO_STATUS 0x0038
#define ADDR_PACKETS_PROC_LO 0x0040
#define ADDR_PACKETS_PROC_HI 0x0044
#define ADDR_PACKETS_ETH_LO 0x0048
#define ADDR_PACKETS_ETH_HI 0x004C
#define ADDR_PACKETS_ICMP_LO 0x0050
#define ADDR_PACKETS_ICMP_HI 0x0054
#define ADDR_PACKETS_UDP_LO 0x0058
#define ADDR_PACKETS_UDP_HI 0x005C
#define ADDR_PACKETS_SLS_LO 0x0060
#define ADDR_PACKETS_SLS_HI 0x0064
#define ADDR_PACKETS_ERR_LEN 0x0068
#define ADDR_PACKETS_ERR_ETH 0x006C
#define ADDR_MAC_ADDR_LO 0x0080
#define ADDR_MAC_ADDR_HI 0x0084
#define ADDR_IPV4_ADDR 0x0088
#define ADDR_NMODULES 0x008C
#define ADDR_DATA_COL_MODE 0x0090
#define ADDR_ONE_OVER_ENERGY 0x0094
#define ADDR_NFRAMES 0x0098
#define ADDR_NSTORAGE_CELLS 0x009C
#define ADDR_MAILBOX_WRDATA 0x00
#define ADDR_MAILBOX_RDDATA 0x08
#define ADDR_MAILBOX_STATUS 0x10
#define ADDR_MAILBOX_SIT 0x18
#define ADDR_MAILBOX_RIT 0x1C
#define MAILBOX_EMPTY (1 << 0)
#define MAILBOX_FULL (1 << 1)
#define MAILBOX_STA (1 << 2)
#define MAILBOX_RTA (1 << 3)
#define CTRL_REGISTER_IDLE (1<<1u)
#endif //DEFINITIONS_H

1255
common/DiffractionExperiment.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

229
common/DiffractionExperiment.h Normal file
View File

@@ -0,0 +1,229 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef DIFFRACTIONEXPERIMENT_H
#define DIFFRACTIONEXPERIMENT_H
#include <chrono>
#include <exception>
#include <mutex>
#include <jfjoch.pb.h>
#include "../compression/CompressionAlgorithmEnum.h"
#include "UnitCell.h"
#include "Coord.h"
#include "Definitions.h"
#include "../frame_serialize/StartMessage.h"
#include "../frame_serialize/EndMessage.h"
enum class DetectorMode : int {
Conversion, Raw, PedestalG0, PedestalG1, PedestalG2
};
class DiffractionExperiment {
JFJochProtoBuf::DatasetSettings dataset;
JFJochProtoBuf::InternalSettings internal;
void CalculateGeometry(int64_t horizontal_stacking, const std::vector<int64_t> &v, int64_t gap_x, int64_t gap_y,
bool mirror_y_in_conversion);
constexpr static const int64_t max_spot_count = 100;
DiffractionExperiment& SetUnitCell(const JFJochProtoBuf::UnitCell &input);
public:
// Public methods are atomic
DiffractionExperiment();
DiffractionExperiment(int64_t horizontal_stacking, const std::vector<int64_t> &v, int64_t gap_x = 0, int64_t gap_y = 0,
bool mirror_y_in_conversion = true);
explicit DiffractionExperiment(const JFJochProtoBuf::JungfraujochSettings &settings);
// Methods below can be chained together
DiffractionExperiment& Mode(DetectorMode input);
DiffractionExperiment& Import(const JFJochProtoBuf::JungfraujochSettings &settings);
DiffractionExperiment& ImagesPerTrigger(int64_t input);
DiffractionExperiment& NumTriggers(int64_t triggers);
DiffractionExperiment& PedestalG0Frames(int64_t input);
DiffractionExperiment& PedestalG1Frames(int64_t input);
DiffractionExperiment& PedestalG2Frames(int64_t input);
DiffractionExperiment& FrameTime(std::chrono::microseconds frame_time,
std::chrono::microseconds in_count_time = std::chrono::microseconds(0));
DiffractionExperiment& Summation(int64_t input);
DiffractionExperiment& ImageTimeUs(std::chrono::microseconds image_time);
DiffractionExperiment& PedestalG1G2FrameTime(std::chrono::microseconds input);
DiffractionExperiment& PhotonEnergy_keV(float input);
DiffractionExperiment& BeamX_pxl(float input);
DiffractionExperiment& BeamY_pxl(float input);
DiffractionExperiment& DetectorDistance_mm(float input);
DiffractionExperiment& ScatteringVector(Coord input);
DiffractionExperiment& ScatteringVector();
DiffractionExperiment& FilePrefix(std::string input);
DiffractionExperiment& DataFileCount(int64_t input);
DiffractionExperiment& Compression(JFJochProtoBuf::Compression input);
DiffractionExperiment& PreviewPeriod(std::chrono::microseconds input);
DiffractionExperiment& SpotFindingPeriod(std::chrono::microseconds input);
DiffractionExperiment& IndexingPeriod(std::chrono::microseconds input);
DiffractionExperiment& UseInternalPacketGenerator(bool input);
DiffractionExperiment& IPv4Subnet(std::string input); // requires 255.255.255.0 subnet
DiffractionExperiment& IPv4Subnet(int64_t input); // requires 255.255.255.0 subnet
DiffractionExperiment& BaseUDPPort(int64_t input);
DiffractionExperiment& MaskModuleEdges(bool input);
DiffractionExperiment& MaskChipEdges(bool input);
DiffractionExperiment& SetUnitCell(const UnitCell &cell);
DiffractionExperiment& SetUnitCell();
DiffractionExperiment& LowResForRadialInt_A(float input);
DiffractionExperiment& HighResForRadialInt_A(float input);
DiffractionExperiment& LowQForRadialInt_recipA(float input);
DiffractionExperiment& HighQForRadialInt_recipA(float input);
DiffractionExperiment& QSpacingForRadialInt_recipA(float input);
DiffractionExperiment& SpaceGroupNumber(int64_t input);
DiffractionExperiment& StorageCells(int64_t input);
DiffractionExperiment& StorageCellStart(int64_t input = 15);
DiffractionExperiment& DetectorType(JFJochProtoBuf::DetectorType type);
DiffractionExperiment& SampleName(std::string input);
DiffractionExperiment& ConversionOnCPU(bool input);
DiffractionExperiment& SourceName(std::string input);
DiffractionExperiment& SourceNameShort(std::string input);
DiffractionExperiment& InstrumentName(std::string input);
DiffractionExperiment& InstrumentNameShort(std::string input);
operator JFJochProtoBuf::JungfraujochSettings() const;
operator JFJochProtoBuf::DetectorInput() const;
void FillMessage(StartMessage &message) const;
JFJochProtoBuf::DetectorConfig DetectorConfig(const JFJochProtoBuf::ReceiverNetworkConfig& net_config) const;
void LoadDatasetSettings(const JFJochProtoBuf::DatasetSettings &settings);
void LoadDetectorSettings(const JFJochProtoBuf::DetectorSettings &settings);
JFJochProtoBuf::DetectorSettings GetDetectorSettings() const;
static void CheckDataProcessingSettings(const JFJochProtoBuf::DataProcessingSettings& settings);
static JFJochProtoBuf::DataProcessingSettings DefaultDataProcessingSettings();
DetectorMode GetDetectorMode() const;
int64_t GetPixelDepth() const;
bool IsPixelSigned() const;
int64_t GetOverflow() const;
int64_t GetUnderflow() const;
int64_t GetNumTriggers() const;
int64_t GetImageNum() const;
int64_t GetImageNumPerTrigger() const;
int64_t GetPedestalG0Frames() const;
int64_t GetPedestalG1Frames() const;
int64_t GetPedestalG2Frames() const;
int64_t GetFrameNum() const;
int64_t GetFrameNumPerTrigger() const;
std::chrono::microseconds GetFrameTime() const;
std::chrono::microseconds GetImageTime() const;
int64_t GetSummation() const;
std::chrono::microseconds GetImageCountTime() const;
std::chrono::microseconds GetFrameCountTime() const;
float GetPhotonEnergy_keV() const;
float GetWavelength_A() const;
float GetBeamX_pxl() const;
float GetBeamY_pxl() const;
float GetDetectorDistance_mm() const;
Coord GetScatteringVector() const;
std::string GetFilePrefix() const;
int64_t GetDataFileCount() const;
JFJochProtoBuf::Compression GetCompressionAlgorithm() const;
CompressionAlgorithm GetCompressionAlgorithmEnum() const;
int64_t GetMaxCompressedSize() const;
int64_t GetDataStreamsNum() const;
int64_t GetModulesNum(uint16_t data_stream = TASK_NO_DATA_STREAM) const;
int64_t GetFirstModuleOfDataStream(uint16_t data_stream) const;
int64_t GetPixelsNum() const;
int64_t GetYPixelsNum() const;
int64_t GetXPixelsNum() const; // X pixels must be the same for all modules
int64_t GetPixel0OfModule(uint16_t module_number) const;
bool IsUpsideDown() const;
std::chrono::microseconds GetPreviewPeriod() const;
int64_t GetPreviewStride() const;
std::chrono::microseconds GetSpotFindingPeriod() const;
int64_t GetSpotFindingStride() const;
std::chrono::microseconds GetIndexingPeriod() const;
int64_t GetIndexingStride() const;
int64_t GetSpotFindingBin() const;
int64_t GetPreviewStride(std::chrono::microseconds period) const;
bool IsUsingInternalPacketGen() const;
uint32_t GetSrcIPv4Address(uint32_t data_stream, uint32_t half_module) const;
uint32_t GetDestIPv4Address(uint32_t data_stream) const;
uint16_t GetDestUDPPort(uint16_t data_stream, uint16_t half_module) const;
bool CheckGitSha1Consistent() const;
std::string CheckGitSha1Msg() const;
bool GetMaskModuleEdges() const;
bool GetMaskChipEdges() const;
UnitCell GetUnitCell() const;
bool HasUnitCell() const;
float ResToPxl(float resolution) const;
Coord LabCoord(float detector_x, float detector_y) const;
float PxlToRes(float detector_x, float detector_y) const;
float GetLowQForRadialInt_recipA() const;
float GetHighQForRadialInt_recipA() const;
float GetQSpacingForRadialInt_recipA() const;
int64_t GetSpaceGroupNumber() const;
int64_t GetStorageCellNumber() const;
int64_t GetStorageCellStart() const;
JFJochProtoBuf::DetectorType GetDetectorType() const;
int64_t GetMaxSpotCount() const;
std::string GetSampleName() const;
bool GetConversionOnCPU() const;
bool GetConversionOnFPGA() const;
DiffractionExperiment& ApplyPixelMaskInFPGA(bool input);
bool GetApplyPixelMaskInFPGA() const;
float GetPixelSize_mm() const;
DiffractionExperiment& Binning2x2(bool input);
bool GetBinning2x2() const;
std::string GetSourceName() const;
std::string GetSourceNameShort() const;
std::string GetInstrumentName() const;
std::string GetInstrumentNameShort() const;
};
inline int64_t CalculateStride(const std::chrono::microseconds &frame_time, const std::chrono::microseconds &preview_time) {
if ((preview_time.count() <= 0) || (frame_time.count() <= 0))
return 0;
else if (preview_time < frame_time)
return 1;
else
return preview_time / frame_time;
}
#endif //DIFFRACTIONEXPERIMENT_H

49
common/DiffractionGeometry.cpp Normal file
View File

@@ -0,0 +1,49 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <cmath>
#include "DiffractionGeometry.h"
DiffractionGeometry::DiffractionGeometry(float in_beam_x_pxl, float in_beam_y_pxl, float in_detector_distance_mm,
float in_pixel_size_mm, float in_wavelength_A, Coord in_scattering_vector)
: beam_x_pxl(in_beam_x_pxl),
beam_y_pxl(in_beam_y_pxl),
detector_distance_mm(in_detector_distance_mm),
pixel_size_mm(in_pixel_size_mm),
wavelength_A(in_wavelength_A),
scattering_vector(in_scattering_vector) {}
Coord DiffractionGeometry::DetectorToLab(float x_pxl, float y_pxl) const {
// Assumes planar detector, 90 deg towards beam
return {(x_pxl - beam_x_pxl) * pixel_size_mm ,
(y_pxl - beam_y_pxl) * pixel_size_mm ,
detector_distance_mm};
}
Coord DiffractionGeometry::DetectorToRecip(float x_pxl, float y_pxl) const {
return DetectorToLab(x_pxl, y_pxl).Normalize() / wavelength_A - scattering_vector;
}
float DiffractionGeometry::PxlToRes(float detector_x, float detector_y) const {
auto lab = DetectorToLab(detector_x, detector_y);
float beam_path = lab.Length();
if (beam_path == detector_distance_mm) return INFINITY;
float cos_2theta = detector_distance_mm / beam_path;
// cos(2theta) = cos(theta)^2 - sin(theta)^2
// cos(2theta) = 1 - 2*sin(theta)^2
// Technically two solutions for two theta, but it makes sense only to take positive one in this case
float sin_theta = sqrt((1-cos_2theta)/2);
return wavelength_A / (2 * sin_theta);
}
float DiffractionGeometry::ResToPxl(float resolution) const {
if (resolution == 0)
return INFINITY;
float sin_theta = wavelength_A / (2 * resolution);
float theta = asin(sin_theta);
float tan_2theta = tan(2 * theta);
return tan_2theta * detector_distance_mm / pixel_size_mm;
}

31
common/DiffractionGeometry.h Normal file
View File

@@ -0,0 +1,31 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef DIFFRACTIONGEOMETRY_H
#define DIFFRACTIONGEOMETRY_H
#include "Coord.h"
#include "DiffractionSpot.h"
class DiffractionGeometry {
const float beam_x_pxl;
const float beam_y_pxl;
const float detector_distance_mm;
const float pixel_size_mm;
const float wavelength_A;
const Coord scattering_vector;
public:
DiffractionGeometry(float beam_x_pxl,
float beam_y_pxl,
float detector_distance_mm,
float pixel_size_mm,
float wavelength_A,
Coord scattering_vector);
Coord DetectorToLab(float x_pxl, float y_pxl) const;
Coord DetectorToRecip(float x_pxl, float y_pxl) const;
float ResToPxl(float resolution) const;
float PxlToRes(float detector_x, float detector_y) const;
};
#endif

68
common/DiffractionSpot.cpp Normal file
View File

@@ -0,0 +1,68 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "DiffractionSpot.h"
#include "RawToConvertedGeometry.h"
DiffractionSpot::DiffractionSpot(uint32_t col, uint32_t line, int64_t in_photons) {
if (in_photons < 0) in_photons = 0;
x = col * static_cast<float>(in_photons);
y = line * static_cast<float>(in_photons);
pixel_count = 1;
photons = in_photons;
max_photons = in_photons;
}
DiffractionSpot& DiffractionSpot::operator+=(const DiffractionSpot &other) {
this->x += other.x;
this->y += other.y;
this->photons += other.photons;
this->max_photons = std::max(this->max_photons, other.max_photons);
this->pixel_count += other.pixel_count;
return *this;
}
int64_t DiffractionSpot::Count() const {
return photons;
}
int64_t DiffractionSpot::MaxCount() const {
return max_photons;
}
Coord DiffractionSpot::RawCoord() const {
return {x / (float)photons, y / (float)photons, 0};
}
int64_t DiffractionSpot::PixelCount() const {
return pixel_count;
}
Coord DiffractionSpot::LabCoord(const DiffractionExperiment &experiment, uint16_t data_stream) const {
return experiment.LabCoord(x / (float)photons, y / (float)photons);
}
Coord DiffractionSpot::ReciprocalCoord(const DiffractionExperiment &experiment, uint16_t data_stream) const {
return LabCoord(experiment, data_stream).Normalize() / experiment.GetWavelength_A()
- experiment.GetScatteringVector();
}
double DiffractionSpot::GetResolution(const DiffractionExperiment &experiment, uint16_t data_stream) const {
return experiment.PxlToRes(x / (float)photons, y / (float)photons);
}
DiffractionSpot::operator SpotToSave() const {
return {
.x = x / static_cast<float>(photons),
.y = y / static_cast<float>(photons),
.intensity = static_cast<float>(photons)
};
}
void DiffractionSpot::AddPixel(uint32_t col, uint32_t line, int64_t photons) {
this->x += col * (float) photons;
this->y += line * (float) photons;
this->photons += photons;
this->max_photons = std::max(this->max_photons, photons);
this->pixel_count += 1;
}

33
common/DiffractionSpot.h Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_DIFFRACTIONSPOT_H
#define JUNGFRAUJOCH_DIFFRACTIONSPOT_H
#include "Coord.h"
#include "DiffractionExperiment.h"
#include "SpotToSave.h"
// Definition of Bragg spot
class DiffractionSpot {
float x;
float y;
int64_t pixel_count;
int64_t photons; // total photon count
int64_t max_photons; // maximum number of counts per pixel in the spot
public:
DiffractionSpot() = default;
DiffractionSpot(uint32_t col, uint32_t line, int64_t photons);
DiffractionSpot& operator+=(const DiffractionSpot& spot);
int64_t PixelCount() const;
int64_t Count() const;
int64_t MaxCount() const;
Coord RawCoord() const;
Coord LabCoord(const DiffractionExperiment &experiment, uint16_t data_stream = TASK_NO_DATA_STREAM) const;
double GetResolution(const DiffractionExperiment &experiment, uint16_t data_stream = TASK_NO_DATA_STREAM) const;
Coord ReciprocalCoord(const DiffractionExperiment &experiment, uint16_t data_stream = TASK_NO_DATA_STREAM) const;
operator SpotToSave() const;
void AddPixel(uint32_t col, uint32_t line, int64_t photons);
};
#endif //JUNGFRAUJOCH_DIFFRACTIONSPOT_H

148
common/FrameTransformation.cpp Normal file
View File

@@ -0,0 +1,148 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <cstring>
#include <bitshuffle/bitshuffle.h>
#include "FrameTransformation.h"
#include "RawToConvertedGeometry.h"
#include "JFJochException.h"
FrameTransformation::FrameTransformation(const DiffractionExperiment &in_experiment) :
experiment(in_experiment), summation(experiment.GetSummation()),
pixel_depth(experiment.GetPixelDepth()), compressor(in_experiment.GetCompressionAlgorithmEnum()),
compression_algorithm(in_experiment.GetCompressionAlgorithmEnum()),
line_shift((experiment.IsUpsideDown() ? -1 : 1) * experiment.GetXPixelsNum()),
binning_2x2(experiment.GetBinning2x2()) {
if ((experiment.GetDetectorMode() == DetectorMode::Conversion) && (summation > 1)) {
for (int i = 0; i < experiment.GetModulesNum(); i++)
summation_buffer.emplace_back(RAW_MODULE_SIZE);
}
precompression_buffer.resize(experiment.GetPixelsNum() * pixel_depth);
if (pixel_depth == 4)
image16bit.resize(experiment.GetPixelsNum(), 0);
if (experiment.GetApplyPixelMaskInFPGA()) {
// Mask gaps
if (pixel_depth == 2) {
auto ptr = (int16_t *) precompression_buffer.data();
for (int i = 0; i < experiment.GetPixelsNum(); i++)
ptr[i] = INT16_MIN;
} else {
auto ptr = (uint32_t *) precompression_buffer.data();
for (int i = 0; i < experiment.GetPixelsNum(); i++)
ptr[i] = INT32_MIN;
}
}
}
FrameTransformation& FrameTransformation::SetOutput(void *output) {
standard_output = (char *) output;
return *this;
}
template <class Td> void AddToFramesSum(Td *destination, const int16_t *source) {
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
if ((source[i] == INT16_MIN) || (destination[i] == INT32_MIN))
destination[i] = INT32_MIN;
else if ((source[i] == INT16_MAX) || (destination[i] == INT32_MAX))
destination[i] = INT32_MAX;
else destination[i] += source[i];
}
}
void FrameTransformation::PackSummation() {
for (int m = 0; m < experiment.GetModulesNum(); m++) {
void *output = precompression_buffer.data() + sizeof(int32_t) * experiment.GetPixel0OfModule(m);
if (binning_2x2)
TransferModuleAdjustMultipixelsBin2x2((int32_t *) output,
(int32_t *) summation_buffer[m].data(),
line_shift,
static_cast<int32_t>(INT32_MIN),
static_cast<int32_t>(INT32_MAX));
else
TransferModuleAdjustMultipixels((int32_t *) output,
(int32_t *) summation_buffer[m].data(),
line_shift,
static_cast<int32_t>(INT32_MIN),
static_cast<int32_t>(INT32_MAX));
for (auto &i: summation_buffer[m])
i = 0;
}
if (pixel_depth == 4) {
// Generate 16-bit preview image
auto arr = (int32_t *) precompression_buffer.data();
for (int pxl = 0; pxl < experiment.GetPixelsNum(); pxl++) {
if (arr[pxl] >= INT16_MAX)
image16bit[pxl] = INT16_MAX;
else if (arr[pxl] <= INT16_MIN)
image16bit[pxl] = INT16_MIN;
else
image16bit[pxl] = static_cast<int16_t>(arr[pxl]);
}
}
}
size_t FrameTransformation::PackStandardOutput() {
if (summation > 1)
PackSummation();
return compressor.Compress(standard_output, precompression_buffer.data(),
experiment.GetPixelsNum(), pixel_depth);
}
void FrameTransformation::ProcessModule(const int16_t *input, size_t frame_number, uint16_t module_number,
int data_stream) {
if (standard_output == nullptr)
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Default stream output not initialized");
size_t module_number_abs = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
if (summation == 1) {
int16_t *output;
output = ((int16_t *) precompression_buffer.data()) + experiment.GetPixel0OfModule(module_number_abs);
if (experiment.GetDetectorMode() != DetectorMode::Conversion)
memcpy(output, input, RAW_MODULE_SIZE * experiment.GetPixelDepth());
else if (binning_2x2)
TransferModuleAdjustMultipixelsBin2x2((int16_t *) output, (int16_t *) input, line_shift,
static_cast<int16_t>(INT16_MIN),
static_cast<int16_t>(INT16_MAX));
else
TransferModuleAdjustMultipixels((int16_t *) output, (int16_t *) input, line_shift,
static_cast<int16_t>(INT16_MIN),
static_cast<int16_t>(INT16_MAX));
} else {
AddToFramesSum(summation_buffer[module_number_abs].data(), input);
}
}
int16_t *FrameTransformation::GetPreview16BitImage() {
if (pixel_depth == 2)
return (int16_t *) precompression_buffer.data();
else
return image16bit.data();
}
void FrameTransformation::ProcessModule(JFConversion &conv, const int16_t *input, size_t frame_number,
uint16_t module_number, int data_stream) {
if (standard_output == nullptr)
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Default stream output not initialized");
size_t module_number_abs = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
auto output = ((int16_t *) precompression_buffer.data()) + experiment.GetPixel0OfModule(module_number_abs);
if (experiment.GetDetectorMode() != DetectorMode::Conversion)
memcpy(output, input, RAW_MODULE_SIZE * experiment.GetPixelDepth());
else
conv.ConvertAdjustGeom((int16_t *) output, (uint16_t *) input, line_shift);
}

40
common/FrameTransformation.h Normal file
View File

@@ -0,0 +1,40 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_FRAMETRANSFORMATION_H
#define JUNGFRAUJOCH_FRAMETRANSFORMATION_H
#include "DiffractionExperiment.h"
#include "../compression/JFJochCompressor.h"
#include "../jungfrau/JFConversion.h"
class FrameTransformation {
const DiffractionExperiment& experiment;
JFJochBitShuffleCompressor compressor;
CompressionAlgorithm compression_algorithm;
std::vector<std::vector<int32_t> > summation_buffer;
std::vector<char> precompression_buffer;
std::vector<int16_t> image16bit;
char *standard_output = nullptr;
const size_t summation;
const size_t pixel_depth;
const int64_t line_shift;
bool binning_2x2;
void PackSummation(); // transfer summed image to converted coordinates, clear summation buffer, no compression
public:
FrameTransformation(const DiffractionExperiment &experiment);
FrameTransformation& SetOutput(void *output);
void ProcessModule(const int16_t *input, size_t frame_number, uint16_t module_number, int data_stream);
void ProcessModule(JFConversion &conv, const int16_t *input, size_t frame_number, uint16_t module_number,
int data_stream);
size_t PackStandardOutput(); // transfer summed image to converted coordinates, clear summation buffer, compress
int16_t *GetPreview16BitImage();
};
#endif //JUNGFRAUJOCH_FRAMETRANSFORMATION_H

506
common/GPUImageAnalysis.cu Normal file
View File

@@ -0,0 +1,506 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "GPUImageAnalysis.h"
#include "JFJochException.h"
#include <sstream>
// input X x Y pixels array
// output X x Y byte array
struct CudaStreamWrapper {
cudaStream_t v;
};
inline void cuda_err(cudaError_t val) {
if (val != cudaSuccess)
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
}
// Determine if pixel could be a spot
// params: spot finding parameters
// val: pixel value
// sum: window sum
// sum2: window sum of squares
// count: window valid pixels count
// return the pixel result: 0-no spot / 1-spot candidate
__device__ __forceinline__ uint8_t pixel_result(const spot_parameters& params, const int64_t val, int64_t sum, int64_t sum2, int64_t count)
{
sum -= val;
sum2 -= val * val;
count -= 1;
const int64_t var = count * sum2 - (sum * sum); // This should be divided by ((2*NBX+1) * (2*NBY+1)-1)*((2*NBX+1) * (2*NBY+1))
const int64_t in_minus_mean = val * count - sum; // Should be divided by ((2*NBX+1) * (2*NBY+1));
const int64_t tmp1 = in_minus_mean * in_minus_mean * (count-1);
const float tmp2 = (var * count) * params.strong_pixel_threshold2;
const bool strong_pixel = (val >= params.count_threshold) & (in_minus_mean > 0) & (tmp1 > tmp2);
return strong_pixel ? 1 : 0;
}
// Find pixels that could be spots
// in: image input values
// out: pixel result byte array, 1 byte per pixel (0:no/1:candidate spot)
// params: spot finding parameters
//
// The algorithm uses multiple waves (blockDim.y) that run over sections of rows.
// Each wave will write output at the back row and read input at the front row.
// Each wave is split into column output sections (blockDim.x)
// A wave section (block) is responsible for a particular row/column section and
// maintains sum/sum2/count values per column for the output row.
// Every cuda thread is associated with a particular column. The thread maintains
// the sum/sum2/count values in shared memory for it's column. To do this, the input
// pixel values for the hight of the aggregation window are saved in shared memory.
__global__ void analyze_pixel(const int16_t *in, uint8_t *out, const spot_parameters params)
{
// assumption: 2 * params.nby + 1 <= params.rows and 2 * params.nbx + 1 <= params.cols
const int32_t window = 2 * (int)params.nby + 1; // vertical window
const int32_t writeSize = blockDim.x - 2 * params.nbx; // output columns per block
const int32_t cmin = blockIdx.x * writeSize; // lowest output column
const int32_t cmax = min(cmin + writeSize, static_cast<int32_t>(params.cols)); // past highest output column
const int32_t col = cmin + threadIdx.x - params.nbx; // thread -> column mapping
const bool data_col = (col >= 0) & (col < static_cast<int32_t>(params.cols)); // read global mem
const bool result_col = (col >= cmin) & (col < cmax); // write result
const int32_t nWaves = gridDim.y; // number of waves
const int32_t rowsPerWave = (params.lines + nWaves - 1) / nWaves; // rows per wave
const int32_t rmin = blockIdx.y * rowsPerWave; // lowest result row for this wave
const int32_t rmax = min(rmin + rowsPerWave, static_cast<int32_t>(params.lines)); // past highest result row for this wave
const int32_t left = max(static_cast<int32_t>(threadIdx.x) - static_cast<int32_t>(params.nbx), 0); // leftmost column touched by this thread
const int32_t right = min(static_cast<int32_t>(threadIdx.x) + static_cast<int32_t>(params.nbx) + 1, static_cast<int32_t>(params.cols)); // past rightmost column touched by this thread
int32_t back = rmin; // back of wave for writing
int32_t front = max(back - static_cast<int32_t>(params.nby), 0); // front of wave for reading (needs to overtake back initially)
extern __shared__ int32_t shared_mem[];
int32_t* shared_sum = shared_mem; // shared buffer [blockDim.x]
int32_t* shared_sum2 = &shared_sum[blockDim.x]; // shared buffer [blockDim.x]
int16_t* shared_count = reinterpret_cast<int16_t*>(&shared_sum2[blockDim.x]); // shared buffer [blockDim.x]
int16_t* shared_val = &shared_count[blockDim.x]; // shared cyclic buffer [window, blockDim.x]
int32_t total_sum; // totals
int32_t total_sum2;
int32_t total_count;
// initialize sum, sum2, count, val buffers
const int16_t ini = params.min_viable_number - 1; // value that is NOT counted
shared_sum[threadIdx.x] = 0; // shared values without effect on totals
shared_sum2[threadIdx.x] = 0;
shared_count[threadIdx.x] = 0;
for (int i=0; i<window; i++)
shared_val[i * blockDim.x + threadIdx.x] = ini;
// wave front up to rmin + nby + 1
do {
if (data_col) { // read at the front end of the wave
const int16_t val = in[front * params.cols + col];
shared_val[(front % window) * blockDim.x + threadIdx.x] = val;
if (val >= params.min_viable_number) {
shared_sum[threadIdx.x] += val;
shared_sum2[threadIdx.x] += val * val;
shared_count[threadIdx.x] += 1;
}
}
front++;
} while (front < rmin + static_cast<int32_t>(params.nby) + 1);
// wave front up to rmax
do {
__syncthreads(); // make others see the shared values
if (result_col) { // write at the back end of the wave
total_sum = total_sum2 = total_count = 0;
for (auto j = left; j < right; j++) {
total_sum += shared_sum[j];
total_sum2 += shared_sum2[j];
total_count += shared_count[j];
}
out[back * params.cols + col] = pixel_result(params, shared_val[(back % window) * blockDim.x + threadIdx.x], total_sum, total_sum2, total_count);
}
back++;
__syncthreads(); // keep shared values until others have seen them
if (data_col) { // read at the front end of the wave
int16_t cnt = 0;
int16_t old = shared_val[(front % window) * blockDim.x + threadIdx.x];
if (old < params.min_viable_number) {
old = 0; // no effect value
cnt = 1; // bring count to normal
}
int16_t val = in[front * params.cols + col];
shared_val[(front % window) * blockDim.x + threadIdx.x] = val;
if (val < params.min_viable_number) {
val = 0; // no effect value
cnt -= 1; // count diff from normal
}
shared_sum[threadIdx.x] += val - old;
shared_sum2[threadIdx.x] += val * val - old * old;
shared_count[threadIdx.x] += cnt;
}
front++;
} while (front < rmax);
// wave back up to rmax
do {
__syncthreads(); // make others see the shared values
if (result_col) { // write at the back end of the wave
total_sum = total_sum2 = total_count = 0;
for (auto j = left; j < right; j++) {
total_sum += shared_sum[j];
total_sum2 += shared_sum2[j];
total_count += shared_count[j];
}
out[back * params.cols + col] = pixel_result(params, shared_val[(back % window) * blockDim.x + threadIdx.x], total_sum, total_sum2, total_count);
}
back++;
__syncthreads(); // keep shared values until others have seen them
if (data_col) { // read at the front end of the wave if possible
int16_t cnt = -1; // normal count diff
int16_t old = shared_val[(front % window) * blockDim.x + threadIdx.x];
if (old < params.min_viable_number) {
old = 0; // no effect value
cnt += 1; // bring count to normal
}
int16_t val = 0;
if (front < params.lines) {
val = in[front * params.cols + col];
if (val < params.min_viable_number)
val = 0; // no effect value
else
cnt += 1; // count diff from normal
}
shared_sum[threadIdx.x] += val - old;
shared_sum2[threadIdx.x] += val * val - old * old;
shared_count[threadIdx.x] += cnt;
}
front++;
} while (back < rmax);
}
__global__ void gpu_radial_integration(const int16_t *image, const uint16_t *rad_integration_mapping,
int32_t *bin_sum, int32_t *bin_count, uint32_t npixel,
uint16_t nbins) {
extern __shared__ int32_t shared_mem[];
int32_t* shared_sum = shared_mem; // shared buffer [nbins]
int32_t* shared_count = &shared_sum[nbins]; // shared buffer [nbins]
uint32_t idx = blockDim.x*blockIdx.x + threadIdx.x;
for (uint32_t i = threadIdx.x; i < 2 * nbins; i += blockDim.x)
shared_mem[i] = 0;
__syncthreads();
for (uint32_t i = idx; i < npixel; i += blockDim.x * gridDim.x) {
uint16_t bin = rad_integration_mapping[i];
int32_t value = image[i];
if ((value > INT16_MIN + 4) && (value < INT16_MAX - 4) && (bin < nbins)) {
atomicAdd(&shared_sum[bin], value);
atomicAdd(&shared_count[bin], 1);
}
}
__syncthreads();
for (uint32_t i = threadIdx.x; i < nbins; i += blockDim.x) {
atomicAdd(&bin_sum[i], shared_sum[i]);
atomicAdd(&bin_count[i], shared_count[i]);
}
}
__global__ void spot_finder_reduce(uint32_t *output, uint32_t* counter, const uint8_t* input,
uint32_t npixel, uint32_t output_size) {
uint32_t idx = blockDim.x*blockIdx.x + threadIdx.x;
if (idx < npixel) {
if (input[idx]) {
auto old_counter = atomicAdd(counter, 1);
if (old_counter < output_size)
output[old_counter] = idx;
}
}
}
__global__ void apply_pixel_mask(int16_t *image, const uint8_t *mask, uint32_t npixel) {
uint32_t idx = blockDim.x*blockIdx.x + threadIdx.x;
if (idx < npixel) {
if (mask[idx] == 0)
image[idx] = INT16_MIN;
}
}
GPUImageAnalysis::GPUImageAnalysis(int32_t in_xpixels, int32_t in_ypixels, const std::vector<uint8_t> &mask,
int32_t gpu_device) :
xpixels(in_xpixels), ypixels(in_ypixels), gpu_out(nullptr), rad_integration_nbins(0) {
int device_count;
cuda_err(cudaGetDeviceCount(&device_count));
if (device_count == 0)
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "No CUDA devices found");
if (gpu_device < 0)
gpu_device = threadid++;
if (device_count > 1)
cuda_err(cudaSetDevice(gpu_device % device_count));
int deviceId;
cuda_err(cudaGetDevice(&deviceId));
cudaDeviceGetAttribute(&numberOfSMs, cudaDevAttrMultiProcessorCount, deviceId);
{
int warp_size;
cuda_err(cudaDeviceGetAttribute(&warp_size, cudaDevAttrWarpSize, deviceId));
}
cudastream = new(CudaStreamWrapper);
cuda_err(cudaStreamCreate(&cudastream->v));
cuda_err(cudaMalloc(&gpu_mask, xpixels * ypixels * sizeof(int8_t)));
cuda_err(cudaMalloc(&gpu_in, xpixels * ypixels * sizeof(int16_t)));
cuda_err(cudaMalloc(&gpu_out, xpixels * ypixels * sizeof(int8_t)));
cuda_err(cudaMalloc(&gpu_out_reduced, maxStrongPixel * sizeof(uint32_t)));
cuda_err(cudaMalloc(&gpu_out_reduced_counter, sizeof(uint32_t)));
cuda_err(cudaHostAlloc(&host_out_reduced, maxStrongPixel * sizeof(uint32_t), cudaHostAllocPortable));
cuda_err(cudaHostAlloc(&host_out_reduced_counter, sizeof(uint32_t), cudaHostAllocPortable));
cuda_err(cudaMemsetAsync(gpu_mask, 1, xpixels*ypixels, cudastream->v));
if (mask.size() != xpixels * ypixels)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in mask size");
cudaMemcpy(gpu_mask, mask.data(), xpixels*ypixels, cudaMemcpyHostToDevice);
}
GPUImageAnalysis::GPUImageAnalysis(int32_t xpixels, int32_t ypixels, const std::vector<uint8_t> &mask,
const std::vector<uint16_t> &rad_int_mapping, uint16_t rad_int_nbins,
int32_t gpu_device) : GPUImageAnalysis(xpixels, ypixels, mask, gpu_device) {
rad_integration_nbins = rad_int_nbins;
if (rad_int_mapping.size() != xpixels * ypixels)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in rad. int. mapping size");
if (rad_integration_nbins > 0) {
cuda_err(cudaMalloc(&gpu_rad_integration_bin_map, xpixels * ypixels * sizeof(int16_t)));
cuda_err(cudaMalloc(&gpu_rad_integration_count, rad_integration_nbins * sizeof(int32_t)));
cuda_err(cudaMalloc(&gpu_rad_integration_sum, rad_integration_nbins * sizeof(int32_t)));
cuda_err(cudaHostAlloc(&host_rad_integration_count, rad_integration_nbins * sizeof(int32_t), cudaHostAllocPortable));
cuda_err(cudaHostAlloc(&host_rad_integration_sum, rad_integration_nbins * sizeof(int32_t), cudaHostAllocPortable));
cudaMemcpy(gpu_rad_integration_bin_map, rad_int_mapping.data(),
xpixels*ypixels * sizeof(uint16_t), cudaMemcpyHostToDevice);
}
}
GPUImageAnalysis::GPUImageAnalysis(int32_t xpixels, int32_t ypixels, const std::vector<uint8_t> &mask,
const RadialIntegrationMapping& mapping,
int32_t gpu_device)
: GPUImageAnalysis(xpixels, ypixels, mask, mapping.GetPixelToBinMapping(),
mapping.GetBinNumber(), gpu_device) {}
GPUImageAnalysis::~GPUImageAnalysis() {
cudaStreamDestroy(cudastream->v);
delete(cudastream);
if (rad_integration_nbins > 0) {
cudaFree(gpu_rad_integration_bin_map);
cudaFree(gpu_rad_integration_count);
cudaFree(gpu_rad_integration_sum);
cudaFreeHost(host_rad_integration_count);
cudaFreeHost(host_rad_integration_sum);
}
cudaFreeHost(host_out_reduced);
cudaFreeHost(host_out_reduced_counter);
cudaFree(gpu_mask);
cudaFree(gpu_in);
cudaFree(gpu_out);
cudaFree(gpu_out_reduced);
cudaFree(gpu_out_reduced_counter);
}
void GPUImageAnalysis::SetInputBuffer(void *ptr) {
host_in = (int16_t *) ptr;
}
bool GPUImageAnalysis::GPUPresent() {
int device_count;
cuda_err(cudaGetDeviceCount(&device_count));
return (device_count > 0);
}
void GPUImageAnalysis::RunSpotFinder(const JFJochProtoBuf::DataProcessingSettings &settings) {
// data_in is CUDA registered memory
// Run COLSPOT (GPU version)
spot_parameters spot_params;
spot_params.strong_pixel_threshold2 = settings.signal_to_noise_threshold() * settings.signal_to_noise_threshold();
spot_params.nbx = settings.local_bkg_size();
spot_params.nby = settings.local_bkg_size();
spot_params.lines = ypixels;
spot_params.cols = xpixels;
spot_params.count_threshold = settings.photon_count_threshold();
spot_params.min_viable_number = INT16_MIN + 5;
if (2 * spot_params.nbx + 1 > windowSizeLimit)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "nbx exceeds window size limit");
if (2 * spot_params.nby + 1 > windowSizeLimit)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "nby exceeds window size limit");
if (windowSizeLimit > numberOfCudaThreads)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "window size limit exceeds number of cuda threads");
if (windowSizeLimit > spot_params.cols)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "window size limit exceeds number of columns");
if (windowSizeLimit > spot_params.lines)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "window size limit exceeds number of lines");
{ // call cuda kernel
const auto nWriters = numberOfCudaThreads - 2 * spot_params.nby;
const auto nBlocks = (spot_params.cols + nWriters - 1) / nWriters;
const auto window = 2 * spot_params.nby + 1;
const auto sharedSize = (2 * sizeof(int32_t) + // sum, sum2
(1 + window) * sizeof(int16_t) // count, val
) * numberOfCudaThreads;
const dim3 blocks(nBlocks, numberOfWaves);
cuda_err(cudaMemsetAsync(gpu_out, 0, xpixels * ypixels * sizeof(uint8_t), cudastream->v));
analyze_pixel<<<blocks, numberOfCudaThreads, sharedSize, cudastream->v>>>
(gpu_in, gpu_out, spot_params);
}
{
cuda_err(cudaMemsetAsync(gpu_out_reduced_counter, 0, sizeof(uint32_t), cudastream->v));
const auto nblocks =
xpixels * ypixels / numberOfCudaThreads + ((xpixels * ypixels % numberOfCudaThreads == 0) ? 0 : 1);
spot_finder_reduce<<<nblocks, numberOfCudaThreads, 0, cudastream->v>>>(gpu_out_reduced, gpu_out_reduced_counter,
gpu_out, xpixels*ypixels, maxStrongPixel);
}
cuda_err(cudaMemcpyAsync(host_out_reduced, gpu_out_reduced, maxStrongPixel * sizeof(uint32_t),
cudaMemcpyDeviceToHost,cudastream->v));
cuda_err(cudaMemcpyAsync(host_out_reduced_counter, gpu_out_reduced_counter, sizeof(uint32_t),
cudaMemcpyDeviceToHost,cudastream->v));
}
void GPUImageAnalysis::GetSpotFinderResults(StrongPixelSet &pixel_set) {
if (host_in == nullptr)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Host/GPU buffer not defined");
cuda_err(cudaStreamSynchronize(cudastream->v));
for (int i = 0; i < std::min<uint32_t>(maxStrongPixel, *host_out_reduced_counter); i++) {
size_t npixel = host_out_reduced[i];
size_t line = npixel / xpixels;
size_t col = npixel % xpixels;
pixel_set.AddStrongPixel(col, line, host_in[npixel]);
}
}
void GPUImageAnalysis::GetSpotFinderResults(const DiffractionExperiment &experiment,
const JFJochProtoBuf::DataProcessingSettings &settings,
std::vector<DiffractionSpot> &vec) {
StrongPixelSet pixel_set(experiment);
GetSpotFinderResults(pixel_set);
pixel_set.FindSpots(experiment, settings, vec);
}
void GPUImageAnalysis::RegisterBuffer() {
cudaHostRegister(host_in, xpixels * ypixels * sizeof(uint16_t), cudaHostRegisterDefault);
}
void GPUImageAnalysis::UnregisterBuffer() {
cudaHostUnregister(host_in);
}
void GPUImageAnalysis::LoadDataToGPU(bool apply_pixel_mask_on_gpu) {
if (host_in == nullptr)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Host/GPU buffer not defined");
cuda_err(cudaMemcpy(gpu_in, host_in, xpixels * ypixels * sizeof(int16_t), cudaMemcpyHostToDevice));
if (apply_pixel_mask_on_gpu) {
const auto nblocks =
xpixels * ypixels / numberOfCudaThreads + ((xpixels * ypixels % numberOfCudaThreads == 0) ? 0 : 1);
apply_pixel_mask<<<nblocks, numberOfCudaThreads, 0, cudastream->v>>>(gpu_in, gpu_mask, xpixels * ypixels);
}
}
void GPUImageAnalysis::RunRadialIntegration() {
if (rad_integration_nbins == 0)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Radial integration not initialized");
cuda_err(cudaMemsetAsync(gpu_rad_integration_sum, 0, rad_integration_nbins * sizeof(int32_t), cudastream->v));
cuda_err(cudaMemsetAsync(gpu_rad_integration_count, 0, rad_integration_nbins * sizeof(int32_t), cudastream->v));
gpu_radial_integration<<<40, numberOfCudaThreads, rad_integration_nbins * sizeof(uint32_t) * 2, cudastream->v>>>(
gpu_in, gpu_rad_integration_bin_map, gpu_rad_integration_sum, gpu_rad_integration_count, xpixels*ypixels,
rad_integration_nbins);
cuda_err(cudaMemcpyAsync(host_rad_integration_count, gpu_rad_integration_count,
rad_integration_nbins * sizeof(int32_t),
cudaMemcpyDeviceToHost, cudastream->v));
cuda_err(cudaMemcpyAsync(host_rad_integration_sum, gpu_rad_integration_sum,
rad_integration_nbins * sizeof(int32_t),
cudaMemcpyDeviceToHost, cudastream->v));
}
void GPUImageAnalysis::GetRadialIntegrationProfile(std::vector<float> &result) {
if (rad_integration_nbins == 0)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Radial integration not initialized");
cuda_err(cudaStreamSynchronize(cudastream->v));
result.resize(rad_integration_nbins);
for (int i = 0; i < rad_integration_nbins; i++) {
if (host_rad_integration_count[i] > 0)
result[i] = static_cast<float>(host_rad_integration_sum[i])
/ static_cast<float>(host_rad_integration_count[i]);
else
result[i] = 0;
}
}
std::vector<int32_t> GPUImageAnalysis::GetRadialIntegrationSum() const {
if (rad_integration_nbins == 0)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Radial integration not initialized");
cuda_err(cudaStreamSynchronize(cudastream->v));
std::vector<int32_t> out(rad_integration_nbins);
memcpy(out.data(), host_rad_integration_sum, rad_integration_nbins * sizeof(int32_t));
return out;
}
std::vector<int32_t> GPUImageAnalysis::GetRadialIntegrationCount() const {
if (rad_integration_nbins == 0)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Radial integration not initialized");
cuda_err(cudaStreamSynchronize(cudastream->v));
std::vector<int32_t> out(rad_integration_nbins);
memcpy(out.data(), host_rad_integration_count, rad_integration_nbins * sizeof(int32_t));
return out;
}
float GPUImageAnalysis::GetRadialIntegrationRangeValue(uint16_t min_bin, uint16_t max_bin) {
if (rad_integration_nbins == 0)
throw JFJochException(JFJochExceptionCategory::SpotFinderError, "Radial integration not initialized");
cuda_err(cudaStreamSynchronize(cudastream->v));
int64_t ret_sum = 0;
int64_t ret_count = 0;
for (int i = std::min(rad_integration_nbins,min_bin);
i <= std::min((uint16_t)(rad_integration_nbins-1),max_bin);
i++) {
ret_sum += host_rad_integration_sum[i];
ret_count += host_rad_integration_count[i];
}
if (ret_count == 0)
return 0;
else
return static_cast<float>(ret_sum) / static_cast<float>(ret_count);
}
std::atomic<uint16_t> GPUImageAnalysis::threadid{0};

82
common/GPUImageAnalysis.h Normal file
View File

@@ -0,0 +1,82 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_GPUIMAGEANALYSIS_H
#define JUNGFRAUJOCH_GPUIMAGEANALYSIS_H
#include <vector>
#include "StrongPixelSet.h"
#include "RadialIntegrationMapping.h"
struct spot_parameters {
float strong_pixel_threshold2;
uint32_t count_threshold;
uint32_t nbx;
uint32_t nby;
uint32_t lines;
uint32_t cols;
int16_t min_viable_number;
};
struct CudaStreamWrapper;
class GPUImageAnalysis {
std::mutex m;
CudaStreamWrapper *cudastream;
static std::atomic<uint16_t> threadid;
const int32_t xpixels;
const int32_t ypixels;
int16_t *host_in = nullptr;
uint8_t *gpu_mask = nullptr;
int16_t *gpu_in = nullptr;
uint8_t *gpu_out = nullptr;
uint32_t *gpu_out_reduced = nullptr;
uint32_t *gpu_out_reduced_counter = nullptr;
uint32_t *host_out_reduced = nullptr;
uint32_t *host_out_reduced_counter = nullptr;
uint16_t rad_integration_nbins;
uint16_t *gpu_rad_integration_bin_map = nullptr;
int32_t *gpu_rad_integration_sum = nullptr;
int32_t *gpu_rad_integration_count = nullptr;
int32_t *host_rad_integration_sum = nullptr, *host_rad_integration_count = nullptr;
int numberOfSMs;
const int numberOfCudaThreads = 128; // #threads per block that works well for Nvidia T4
const int numberOfWaves = 40; // #waves that works well for Nvidia T4
const int windowSizeLimit = 21; // limit on the window size (2nby+1, 2nbx+1) to prevent shared memory problems
const int maxStrongPixel = 65536;
public:
GPUImageAnalysis(int32_t xpixels, int32_t ypixels, const std::vector<uint8_t> &mask, int32_t gpu_device = -1);
GPUImageAnalysis(int32_t xpixels, int32_t ypixels, const std::vector<uint8_t> &mask,
const std::vector<uint16_t> &rad_int_mapping, uint16_t rad_int_nbins,
int32_t gpu_device = -1);
GPUImageAnalysis(int32_t xpixels, int32_t ypixels, const std::vector<uint8_t> &mask,
const RadialIntegrationMapping& mapping,int32_t gpu_device = -1);
~GPUImageAnalysis();
void SetInputBuffer(void *host_in);
void LoadDataToGPU(bool apply_pixel_mask_on_gpu = true);
void RunSpotFinder(const JFJochProtoBuf::DataProcessingSettings &settings);
void GetSpotFinderResults(StrongPixelSet &pixel_set);
void GetSpotFinderResults(const DiffractionExperiment &experiment, const JFJochProtoBuf::DataProcessingSettings &settings,
std::vector<DiffractionSpot> &vec);
void RunRadialIntegration();
void GetRadialIntegrationProfile(std::vector<float> &result);
[[nodiscard]] std::vector<int32_t> GetRadialIntegrationSum() const;
[[nodiscard]] std::vector<int32_t> GetRadialIntegrationCount() const;
[[nodiscard]] float GetRadialIntegrationRangeValue(uint16_t min_bin, uint16_t max_bin);
static bool GPUPresent();
void RegisterBuffer();
void UnregisterBuffer();
};
#endif //JUNGFRAUJOCH_GPUIMAGEANALYSIS_H

12
common/GitInfo.cpp.in Normal file
View File

@@ -0,0 +1,12 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <string>
std::string jfjoch_git_sha1() {
return "@GIT_SHA1@";
}
std::string jfjoch_git_date() {
return "@GIT_DATE@";
}

12
common/GitInfo.h Normal file
View File

@@ -0,0 +1,12 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_GITINFO_H
#define JUNGFRAUJOCH_GITINFO_H
#include <string>
std::string jfjoch_git_sha1();
std::string jfjoch_git_date();
#endif //JUNGFRAUJOCH_GITINFO_H

29
common/ImagePusher.cpp Normal file
View File

@@ -0,0 +1,29 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "ImagePusher.h"
void PrepareCBORImage(DataMessage& message,
const DiffractionExperiment &experiment,
void *image, size_t image_size) {
message.image.data = (uint8_t *) image;
message.image.size = image_size;
message.image.xpixel = experiment.GetXPixelsNum();
message.image.ypixel = experiment.GetYPixelsNum();
message.image.pixel_depth_bytes = experiment.GetPixelDepth();
message.image.pixel_is_signed = experiment.IsPixelSigned();
message.image.algorithm = experiment.GetCompressionAlgorithmEnum();
message.image.channel = "default";
}
void PrepareDataMessageSpots(DataMessage& message,
const std::vector<DiffractionSpot>& spots) {
message.spots.clear();
for (const auto & spot : spots)
message.spots.push_back(spot);
}
void ImagePusher::SendData(const std::vector<uint8_t> &serialized_image, int64_t image_number) {
SendDataInternal(serialized_image, image_number);
}

30
common/ImagePusher.h Normal file
View File

@@ -0,0 +1,30 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_IMAGEPUSHER_H
#define JUNGFRAUJOCH_IMAGEPUSHER_H
#include <cstdint>
#include <vector>
#include "DiffractionExperiment.h"
#include "DiffractionSpot.h"
#include "../frame_serialize/JFJochFrameSerializer.h"
#include "../frame_serialize/StartMessage.h"
#include "../frame_serialize/EndMessage.h"
void PrepareCBORImage(DataMessage& message,
const DiffractionExperiment &experiment,
void *image, size_t image_size);
class ImagePusher {
protected:
virtual void SendDataInternal(const std::vector<uint8_t>& serialized_image, int64_t image_number) = 0;
public:
virtual void StartDataCollection(const StartMessage& message) = 0;
virtual void EndDataCollection(const EndMessage& message) = 0;
void SendData(const std::vector<uint8_t>& serialized_image, int64_t image_number);
};
#endif //JUNGFRAUJOCH_IMAGEPUSHER_H

146
common/JFJochException.h Normal file
View File

@@ -0,0 +1,146 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef SLSEXCEPTION_H
#define SLSEXCEPTION_H
#include <string>
#include <exception>
enum class JFJochExceptionCategory {
JSON,
InputParameterAboveMax,
InputParameterBelowMin,
InputParameterInvalid,
WrongNumber,
WrongUnit,
ArrayOutOfBounds,
HDF5,
IBVerbs,
RESTCommandUnknown,
WrongDAQState,
Detector,
MemAllocFailed,
OpenCAPIError,
GPUCUDAError,
GainFileOpenError,
MockFileOpenError,
Compression,
SharedMemory,
ZeroMQ,
ServiceUndefined,
TriggerError,
FileWriteError,
ZSTDCompressionError,
LogstashConnectionFailure,
TIFFGeneratorError,
gRPCError,
PreviewError,
HardwareParityError,
SpotFinderError,
PCIeError,
CBORError,
UDPError,
CommunicationError
};
class JFJochException : public std::exception {
std::string msg;
JFJochExceptionCategory category;
static std::string DecodeCategory(JFJochExceptionCategory category) {
switch (category) {
case JFJochExceptionCategory::JSON:
return "JSON parsing or analysis error";
case JFJochExceptionCategory::InputParameterAboveMax:
return "Input parameter above max";
case JFJochExceptionCategory::InputParameterBelowMin:
return "Input parameter below min";
case JFJochExceptionCategory::InputParameterInvalid:
return "Input parameter invalid";
case JFJochExceptionCategory::WrongNumber:
return "Wrong number";
case JFJochExceptionCategory::WrongUnit:
return "Wrong unit";
case JFJochExceptionCategory::ArrayOutOfBounds:
return "Array out of bounds";
case JFJochExceptionCategory::HDF5:
return "HDF5 error";
case JFJochExceptionCategory::IBVerbs:
return "Infiniband error";
case JFJochExceptionCategory::RESTCommandUnknown:
return "REST command unknown";
case JFJochExceptionCategory::WrongDAQState:
return "DAQ state error";
case JFJochExceptionCategory::Detector:
return "Detector error";
case JFJochExceptionCategory::MemAllocFailed:
return "Memory error";
case JFJochExceptionCategory::OpenCAPIError:
return "OpenCAPI error";
case JFJochExceptionCategory::GPUCUDAError:
return "CUDA (GPU) error";
case JFJochExceptionCategory::GainFileOpenError:
return "Gain file open error";
case JFJochExceptionCategory::MockFileOpenError:
return "Mock file open error";
case JFJochExceptionCategory::Compression:
return "Compression error";
case JFJochExceptionCategory::SharedMemory:
return "Shared memory error";
case JFJochExceptionCategory::ZeroMQ:
return "ZeroMQ error";
case JFJochExceptionCategory::ServiceUndefined:
return "Service undefined error";
case JFJochExceptionCategory::TriggerError:
return "Trigger error";
case JFJochExceptionCategory::FileWriteError:
return "File writer error";
case JFJochExceptionCategory::ZSTDCompressionError:
return "Zstd compressor error";
case JFJochExceptionCategory::LogstashConnectionFailure:
return "Logstash connection error";
case JFJochExceptionCategory::TIFFGeneratorError:
return "TIFF writer error";
case JFJochExceptionCategory::gRPCError:
return "Remote node (gRPC) error";
case JFJochExceptionCategory::PreviewError:
return "Preview error";
case JFJochExceptionCategory::HardwareParityError:
return "Parity error (HW)";
case JFJochExceptionCategory::SpotFinderError:
return "Spot finder";
case JFJochExceptionCategory::PCIeError:
return "PCIe driver error";
case JFJochExceptionCategory::CBORError:
return "CBOR serialization error";
case JFJochExceptionCategory::UDPError:
return "UDP simulator error";
case JFJochExceptionCategory::CommunicationError:
return "Communication error";
default:
return "";
}
}
public:
JFJochException() = default;
JFJochException(JFJochExceptionCategory in_val, const std::string &description, int optional = 0) noexcept :
category(in_val) {
try {
msg = DecodeCategory(category) + " (" + description + ")";
} catch (...) {}
}
JFJochException(const JFJochException &e) noexcept {
try {
category = e.category;
msg = e.msg;
} catch (...) {}
}
[[nodiscard]] const char *what() const noexcept override {
return msg.c_str();
}
};
#endif //SLSEXCEPTION_H

21
common/Latch.cpp Normal file
View File

@@ -0,0 +1,21 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "Latch.h"
Latch::Latch(uint32_t in_count) {
count = in_count;
}
void Latch::CountDown() {
std::unique_lock<std::mutex> ul(m);
if (count != 0)
count--;
if (count == 0)
c.notify_all();
}
void Latch::Wait() {
std::unique_lock<std::mutex> ul(m);
c.wait(ul, [&]{return count == 0;});
}

21
common/Latch.h Normal file
View File

@@ -0,0 +1,21 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_LATCH_H
#define JUNGFRAUJOCH_LATCH_H
#include <mutex>
#include <condition_variable>
class Latch {
std::mutex m;
std::condition_variable c;
uint32_t count;
public:
Latch(uint32_t count);
void CountDown();
void Wait();
};
#endif //JUNGFRAUJOCH_LATCH_H

33
common/Logger.cpp Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "spdlog/sinks/daily_file_sink.h"
#include "spdlog/sinks/stdout_color_sinks.h"
#include "spdlog/fmt/bin_to_hex.h"
#include "Logger.h"
#include "GitInfo.h"
Logger::Logger(const std::string &service_name, const std::string &file_name) {
std::vector<spdlog::sink_ptr> sinks;
sinks.push_back(std::make_shared<spdlog::sinks::stdout_color_sink_mt>());
if (!file_name.empty())
sinks.push_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(file_name, 23, 59));
spdlog_logger = std::make_shared<spdlog::logger>(service_name, std::begin(sinks), std::end(sinks));
spdlog_logger->info("Git sha {} ({})", jfjoch_git_sha1().substr(0, 6), jfjoch_git_date());
}
void Logger::ErrorException(const std::exception &e) {
spdlog_logger->error(e.what());
}
Logger &Logger::Verbose(bool input) {
if (input)
spdlog_logger->set_level(spdlog::level::debug);
else
spdlog_logger->set_level(spdlog::level::info);
return *this;
}

59
common/Logger.h Normal file
View File

@@ -0,0 +1,59 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_LOGGER_H
#define JUNGFRAUJOCH_LOGGER_H
#include <cstdint>
#include <chrono>
#include <iostream>
#include <mutex>
#include <memory>
#include "spdlog/spdlog.h"
#include "spdlog/fmt/fmt.h"
namespace spdlog {
class logger;
}
class Logger {
mutable std::mutex m;
std::shared_ptr<spdlog::logger> spdlog_logger;
std::string service;
std::string hostname;
public:
Logger(const std::string &service_name, const std::string &file_name = "");
void ErrorException(const std::exception &e);
void Info(const std::string& msg) { spdlog_logger->info(msg); }
void Warning(const std::string& msg) { spdlog_logger->warn(msg); }
void Error(const std::string& msg) { spdlog_logger->error(msg); }
void Debug(const std::string& msg) { spdlog_logger->debug(msg); }
template<typename... Args>
void Info(fmt::format_string<Args...> fmt, Args &&... args) {
spdlog_logger->info(fmt, std::forward<Args>(args)...);
}
template<typename... Args>
void Error(fmt::format_string<Args...> fmt, Args &&... args) {
spdlog_logger->error(fmt, std::forward<Args>(args)...);
}
template<typename... Args>
void Debug(fmt::format_string<Args...> fmt, Args &&... args) {
spdlog_logger->debug(fmt, std::forward<Args>(args)...);
}
template<typename... Args>
void Warning(fmt::format_string<Args...> fmt, Args &&... args) {
spdlog_logger->warn(fmt, std::forward<Args>(args)...);
}
Logger& Verbose(bool input);
};
#endif //JUNGFRAUJOCH_LOGGER_H

50
common/NetworkAddressConvert.cpp Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <arpa/inet.h>
#include <sstream>
#include <iomanip>
#include "NetworkAddressConvert.h"
#include "JFJochException.h"
uint32_t IPv4AddressFromStr(const std::string& addr) {
in_addr ipv4_addr{};
if (inet_pton(AF_INET, addr.c_str(), &ipv4_addr) != 1)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Need proper IPv4 address");
return ipv4_addr.s_addr;
}
std::string IPv4AddressToStr(uint32_t addr) {
char tmp[256];
if (inet_ntop(AF_INET, &addr, tmp, 255) != tmp)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Error in exporting IPv4 address");
return {tmp};
}
std::string MacAddressToStr(uint64_t addr) {
uint64_t mac_address_network_order = addr;
std::ostringstream ss;
for (int i = 0; i < 6; i++) {
ss << std::setfill('0') << std::setw(2) << std::hex << ((mac_address_network_order >> (i * 8)) & 0xFF);
if (i != 5) ss << ":";
}
return ss.str();
}
uint64_t MacAddressFromStr(const std::string &addr) {
uint64_t ret = 0;
uint32_t values[6];
char end_char;
if( sscanf( addr.c_str(), "%x:%x:%x:%x:%x:%x%c",
&values[0], &values[1], &values[2],
&values[3], &values[4], &values[5], &end_char ) != 6 )
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "MAC address invalid");
for (int i = 0; i < 6; i++) {
if (values[i] > 255)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "MAC address out of range");
ret |= (uint64_t)values[i] << (i*8);
}
return ret;
}

15
common/NetworkAddressConvert.h Normal file
View File

@@ -0,0 +1,15 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_NETWORKADDRESSCONVERT_H
#define JUNGFRAUJOCH_NETWORKADDRESSCONVERT_H
#include <cstdint>
#include <string>
std::string IPv4AddressToStr(uint32_t addr);
std::string MacAddressToStr(uint64_t addr);
uint32_t IPv4AddressFromStr(const std::string& addr);
uint64_t MacAddressFromStr(const std::string& addr);
#endif //JUNGFRAUJOCH_NETWORKADDRESSCONVERT_H

74
common/RadialIntegration.cpp Normal file
View File

@@ -0,0 +1,74 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "RadialIntegration.h"
#include "JFJochException.h"
RadialIntegration::RadialIntegration(const std::vector<uint16_t>& in_mapping, uint16_t in_nbins) :
pixel_to_bin(in_mapping), nbins(in_nbins), sum(in_nbins, 0), count(in_nbins, 0)
{}
RadialIntegration::RadialIntegration(const RadialIntegrationMapping &mapping) :
RadialIntegration(mapping.GetPixelToBinMapping(), mapping.GetBinNumber())
{}
void RadialIntegration::Clear() {
for (auto &i : sum)
i = 0;
for (auto &i : count)
i = 0;
}
void RadialIntegration::Process(const int16_t *data, size_t npixel) {
if (npixel != pixel_to_bin.size())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Mismatch in size of pixel-to-bin mapping and image");
for (int i = 0; i < npixel; i++) {
auto bin = pixel_to_bin[i];
auto value = data[i];
if ((value > INT16_MIN + 4) && (value < INT16_MAX - 4) && (bin < nbins)) {
sum[bin] += value;
count[bin] += 1;
}
}
}
void RadialIntegration::ProcessOneImage(const int16_t *data, size_t npixel) {
Clear();
Process(data, npixel);
}
void RadialIntegration::GetResult(std::vector<float> &result) const {
result.resize(nbins);
for (int i = 0; i < nbins; i++) {
if (count[i] > 0)
result[i] = static_cast<float>(sum[i]) / static_cast<float>(count[i]);
else
result[i] = 0;
}
}
const std::vector<int64_t> &RadialIntegration::GetSum() const {
return sum;
}
const std::vector<int64_t> &RadialIntegration::GetCount() const {
return count;
}
float RadialIntegration::GetRangeValue(uint16_t min_bin, uint16_t max_bin) {
int64_t ret_sum = 0;
int64_t ret_count = 0;
for (int i = std::min(nbins,min_bin); i <= std::min((uint16_t)(nbins-1),max_bin); i++) {
ret_sum += sum[i];
ret_count += count[i];
}
if (ret_count == 0)
return 0;
else
return static_cast<float>(ret_sum) / static_cast<float>(ret_count);
}

32
common/RadialIntegration.h Normal file
View File

@@ -0,0 +1,32 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_RADIALINTEGRATION_H
#define JUNGFRAUJOCH_RADIALINTEGRATION_H
#include <vector>
#include <cstdint>
#include <cmath>
#include "DiffractionExperiment.h"
#include "RadialIntegrationMapping.h"
class RadialIntegration {
const std::vector<uint16_t>& pixel_to_bin;
const uint16_t nbins;
std::vector<int64_t> sum;
std::vector<int64_t> count;
public:
RadialIntegration(const RadialIntegrationMapping& mapping);
RadialIntegration(const std::vector<uint16_t>& mapping, uint16_t nbins);
void Clear();
void Process(const int16_t *data, size_t npixel);
void ProcessOneImage(const int16_t *data, size_t npixel); // Process + Clear
void GetResult(std::vector<float> &result) const;
[[nodiscard]] float GetRangeValue(uint16_t min_bin, uint16_t max_bin);
[[nodiscard]] const std::vector<int64_t>& GetSum() const;
[[nodiscard]] const std::vector<int64_t>& GetCount() const;
};
#endif //JUNGFRAUJOCH_RADIALINTEGRATION_H

73
common/RadialIntegrationMapping.cpp Normal file
View File

@@ -0,0 +1,73 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <cmath>
#include "RadialIntegrationMapping.h"
#include "JFJochException.h"
RadialIntegrationMapping::RadialIntegrationMapping(const DiffractionExperiment& experiment, const uint8_t *one_byte_mask) :
low_q(experiment.GetLowQForRadialInt_recipA()),
high_q(experiment.GetHighQForRadialInt_recipA()),
q_spacing(experiment.GetQSpacingForRadialInt_recipA()),
pixel_to_bin(experiment.GetPixelsNum()),
max_bin_number(0)
{
if (q_spacing < 0.0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Q-spacing has to be >= 0.0");
if (low_q >= high_q)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Low-Q must be smaller than high-Q");
if (low_q <= 0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Low-Q must be > 0");
if (std::ceil((high_q-low_q) / q_spacing ) >= UINT16_MAX)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Cannot accommodate more than 65536 rings");
for (int y = 0; y < experiment.GetYPixelsNum(); y++) {
for (int x = 0; x < experiment.GetXPixelsNum(); x++) {
int64_t pixel_number = y * experiment.GetXPixelsNum() + x;
double pixel_q = 2 * M_PI / experiment.PxlToRes(x, y);
if (((one_byte_mask != nullptr) && (one_byte_mask[pixel_number] == 0))
|| (pixel_q < low_q)
|| (pixel_q >= high_q))
pixel_to_bin[pixel_number] = UINT16_MAX;
else
pixel_to_bin[pixel_number] = std::floor((pixel_q - low_q) / q_spacing);
}
}
// In principle max bin number is equal to std::floor((high_q - low_q) / q_spacing), but it might be lower than this
// depending on detector distance and beam center position
for (const auto &i: pixel_to_bin) {
if ((i != UINT16_MAX) && (i > max_bin_number))
max_bin_number = i;
}
bin_to_q.resize(max_bin_number + 1);
for (int i = 0; i < max_bin_number + 1; i++)
bin_to_q[i] = static_cast<float>(q_spacing * (i + 0.5) + low_q);
}
uint16_t RadialIntegrationMapping::GetBinNumber() const {
return max_bin_number + 1;
}
const std::vector<uint16_t> &RadialIntegrationMapping::GetPixelToBinMapping() const {
return pixel_to_bin;
}
const std::vector<float> &RadialIntegrationMapping::GetBinToQ() const {
return bin_to_q;
}
double RadialIntegrationMapping::QToBin(double q) const {
return std::min(static_cast<double>(max_bin_number), std::max(0.0, (q - low_q) / q_spacing));
}

24
common/RadialIntegrationMapping.h Normal file
View File

@@ -0,0 +1,24 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_RADIALINTEGRATIONMAPPING_H
#define JUNGFRAUJOCH_RADIALINTEGRATIONMAPPING_H
#include <optional>
#include "DiffractionExperiment.h"
class RadialIntegrationMapping {
const double low_q, high_q, q_spacing;
std::vector<float> bin_to_q;
std::vector<uint16_t> pixel_to_bin;
uint16_t max_bin_number;
public:
RadialIntegrationMapping(const DiffractionExperiment& experiment, const uint8_t *one_byte_mask = nullptr);
[[nodiscard]] uint16_t GetBinNumber() const;
[[nodiscard]] const std::vector<uint16_t> &GetPixelToBinMapping() const;
[[nodiscard]] const std::vector<float> &GetBinToQ() const;
[[nodiscard]] double QToBin(double q) const;
};
#endif //JUNGFRAUJOCH_RADIALINTEGRATIONMAPPING_H

298
common/RawToConvertedGeometry.h Normal file
View File

@@ -0,0 +1,298 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_RAWTOCONVERTEDGEOMETRY_H
#define JUNGFRAUJOCH_RAWTOCONVERTEDGEOMETRY_H
#include <cmath>
#include "DiffractionExperiment.h"
// Take half of the number, but only if not bad pixel/overload
template <typename T> T half(T in, T min, T max) {
T tmp = in;
if ((in > min) && (in < max)) tmp /= 2;
return tmp;
}
template <typename T> T quarter(T in, T min, T max) {
T tmp = in;
if ((in > min) && (in < max)) tmp /= 4;
return tmp;
}
// Copy line, divide everything by 2 and extend multipixels + divide them by additional factor of 2
template <typename Td, typename Ts> void LineCopyAndAdjustMultipixelMidRow(Td *destination, const Ts* source, Ts min, Ts max) {
for (int chip = 0; chip < 4; chip++) {
for (int i = 0; i < 256; i++) {
destination[i+chip*258] = half(source[i+chip*256], min, max);
}
}
for (int i = 0; i < 3; i++) {
destination[255+i*258] = quarter(source[255 + i*256], min, max);
destination[256+i*258] = quarter(source[255 + i*256], min, max);
destination[257+i*258] = quarter(source[256 + i*256], min, max);
destination[258+i*258] = quarter(source[256 + i*256], min, max);
}
}
// Copy line and extend multipixels + divide them by 2
template <typename Td, typename Ts> void LineCopyAndAdjustMultipixel(Td *destination, const Ts* source, Ts min, Ts max) {
for (int chip = 0; chip < 4; chip++) {
for (int i = 0; i < 256; i++) {
destination[i + chip * 258] = source[i + chip * 256];
}
}
for (int chip = 0; chip < 3; chip++) {
destination[255+chip*258] = half(source[255 + chip*256], min, max);
destination[256+chip*258] = half(source[255 + chip*256], min, max);
destination[257+chip*258] = half(source[256 + chip*256], min, max);
destination[258+chip*258] = half(source[256 + chip*256], min, max);
}
}
// Copy line and copy multipixels (e.g. for mask)
template <typename Td, typename Ts> void LineCopyAndAddMultipixel(Td *destination, const Ts* source) {
for (int chip = 0; chip < 4; chip++) {
for (int i = 0; i < 256; i++)
destination[i+chip*258] = source[i+chip*256];
}
for (int i = 0; i < 3; i++) {
destination[256+i*258] = source[255 + i*256];
destination[257+i*258] = source[256 + i*256];
}
}
template <class T, class Tint = int32_t>
T Bin2x2_sum(T a, T b, T c, T d, T underload, T overload) {
T ret;
if ((a <= underload) || (b <= underload) || (c <= underload) || (d <= underload))
ret = underload;
else if ((a >= overload) || (b >= overload) || (c >= overload) || (d >= overload))
ret = overload;
else {
Tint sum = a + b + c +d;
if (sum > overload)
ret = overload;
else
ret = static_cast<T>(sum);
}
return ret;
}
// Copy line and copy multipixels (e.g. for mask)
template <class Ts, class Tint = int32_t>
void LineCopyAndAdjustMultipixelBin2x2(Ts *destination, const Ts* source, Ts min, Ts max) {
destination[0] = Bin2x2_sum<Ts, Tint>(source[0], source[1], source[1024], source[1025], min, max);
for (int chip = 0; chip < 4; chip++) {
for (int i = 1; i < 127; i++)
destination[i + 129 * chip] = Bin2x2_sum<Ts, Tint>(source[2 * i + 256*chip],
source[2 * i + 1 + 256*chip],
source[2 * i + 1024 + 256*chip],
source[2 * i + 1025 + 256*chip],
min, max);
}
destination[514] = Bin2x2_sum<Ts, Tint>(source[1022], source[1023], source[2046], source[2047], min, max);
for (int chip = 0; chip < 3; chip++) {
destination[127 + chip * 129] = Bin2x2_sum<Ts, Tint>(source[254 + chip * 256],
half(source[255 + chip * 256], min, max),
source[254 + chip * 256 + 1024],
half(source[255 + chip * 256 + 1024], min, max),
min, max);
destination[128 + chip * 129] = Bin2x2_sum<Ts, Tint>(half(source[255 + chip * 256], min, max),
half(source[256 + chip * 256], min, max),
half(source[255 + chip * 256 + 1024], min, max),
half(source[256 + chip * 256 + 1024], min, max),
min, max);
destination[129 + chip * 129] = Bin2x2_sum<Ts, Tint>(source[257 + chip * 256],
half(source[256 + chip * 256], min, max),
source[257 + chip * 256 + 1024],
half(source[256 + chip * 256 + 1024], min, max),
min, max);
};
}
template <class Ts, class Tint = int32_t>
void LineCopyAndAdjustMultipixelBin2x2MidRow(Ts *destination, const Ts* source, Ts min, Ts max) {
destination[0] = Bin2x2_sum<Ts, Tint>(source[0], source[1], 0, 0, min, max);
for (int chip = 0; chip < 4; chip++) {
for (int i = 1; i < 127; i++)
destination[i + 129*chip] = Bin2x2_sum<Ts, Tint>(source[2 * i + 256*chip],
source[2 * i + 1 + 256*chip],
0,
0,
min, max);
}
destination[514] = Bin2x2_sum<Ts, Tint>(source[1022], source[1023], 0, 0, min, max);
for (int chip = 0; chip < 3; chip++) {
destination[127+chip*129] = Bin2x2_sum<Ts, Tint>(source[254 + chip * 256],
half(source[255 + chip * 256], min, max),
0,
0,
min, max);
destination[128+chip*129] = Bin2x2_sum<Ts, Tint>(half(source[255 + chip * 256], min, max),
half(source[256 + chip * 256], min, max),
0,
0,
min, max);
destination[129+chip*129] = Bin2x2_sum<Ts, Tint>(source[257 + chip * 256],
half(source[256 + chip * 256], min, max),
0,
0,
min, max);
}
}
template <class Td, class Ts> void TransferModule(Td *destination, const Ts *source, int64_t line_shift) {
for (size_t line = 0; line < RAW_MODULE_LINES; line++) {
if ((line == 255) || (line == 256)) {
LineCopyAndAddMultipixel<Td, Ts>(destination + line_shift * (line + 1), source + RAW_MODULE_COLS * line);
LineCopyAndAddMultipixel<Td, Ts>(destination + line_shift * (line + ((line > 255) ? 2 : 0)), source + RAW_MODULE_COLS * line);
} else {
LineCopyAndAddMultipixel<Td, Ts>(destination + line_shift * (line + ((line > 255) ? 2 : 0)), source + RAW_MODULE_COLS * line);
}
}
}
template <class Td, class Ts> void TransferModuleAdjustMultipixels(Td *destination, const Ts *source, int64_t line_shift, Ts min, Ts max) {
for (size_t line = 0; line < RAW_MODULE_LINES; line++) {
if ((line != 255) && (line != 256))
LineCopyAndAdjustMultipixel<Td, Ts>(destination + line_shift * (line + ((line > 255) ? 2 : 0)),
source + RAW_MODULE_COLS * line, min, max);
else {
LineCopyAndAdjustMultipixelMidRow<Td, Ts>(destination + line_shift * (line + 1),
source + RAW_MODULE_COLS * line, min, max);
memcpy(destination + line_shift * (line + ((line > 255) ? 2 : 0)),
destination + line_shift * (line + 1),CONVERTED_MODULE_COLS*sizeof(Td));
}
}
}
template <class Ts, class Tint = int32_t>
void TransferModuleAdjustMultipixelsBin2x2(Ts *destination, const Ts *source,
int64_t line_shift, Ts min, Ts max) {
for (size_t line = 0; line < RAW_MODULE_LINES/2; line++) {
if (line == 127)
LineCopyAndAdjustMultipixelBin2x2MidRow<Ts, Tint>(destination + line_shift * 128,
source + 255 * RAW_MODULE_COLS, min, max);
if (line == 128)
LineCopyAndAdjustMultipixelBin2x2MidRow<Ts, Tint>(destination + line_shift * 129,
source + 256 * RAW_MODULE_COLS, min, max);
else
LineCopyAndAdjustMultipixelBin2x2<Ts, Tint>(destination + line_shift * (line + ((line > 127) ? 1 : 0)),
source + line * 2 * RAW_MODULE_COLS, min, max);
}
}
inline std::pair<int64_t, int64_t> RawToConvertedCoordinate(const DiffractionExperiment& experiment,
uint32_t module_number,
uint32_t pixel_within_module) {
int64_t line0 = experiment.GetPixel0OfModule(module_number) / experiment.GetXPixelsNum();
int64_t col0 = experiment.GetPixel0OfModule(module_number) % experiment.GetXPixelsNum();
int64_t line = pixel_within_module / RAW_MODULE_COLS;
int64_t col = pixel_within_module % RAW_MODULE_COLS;
line += (line / 256) * 2;
col += (col / 256) * 2;
return {col0 + col, line0 + (experiment.IsUpsideDown() ? -1 : 1) * line};
}
inline Coord RawToConvertedCoordinate(const DiffractionExperiment& experiment, uint16_t data_stream, const Coord &raw) {
if (experiment.GetDetectorMode() == DetectorMode::Conversion) {
size_t module_number = experiment.GetFirstModuleOfDataStream(data_stream) + raw.y / RAW_MODULE_LINES;
double line = raw.y - floor(raw.y / RAW_MODULE_LINES) * RAW_MODULE_LINES;
double col = raw.x;
line += floor(line / 256) * 2;
col += floor(col / 256) * 2;
Coord out(experiment.GetPixel0OfModule(module_number) % experiment.GetXPixelsNum(),
experiment.GetPixel0OfModule(module_number) / experiment.GetXPixelsNum(),
raw.z);
out.x += col;
out.y += (experiment.IsUpsideDown() ? -1 : 1) * line;
return out;
} else return raw;
}
// Input coord is column + 1024 * (line + 512 * module)
// Copies result over multipixel - can be used for example for mask calculation
template <class Td, class Ts> void RawToConvertedGeometry(const DiffractionExperiment& experiment, Td *destination, const Ts *source) {
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++)
TransferModule<Td, Ts>(destination + experiment.GetPixel0OfModule(module_number),
source + module_number * RAW_MODULE_SIZE,
(experiment.IsUpsideDown() ? -1 : 1) * experiment.GetXPixelsNum());
}
// Input coord is column + 1024 * (line + 512 * module)
template <class Ts> void RawToConvertedGeometryAdjustMultipixels(const DiffractionExperiment& experiment, Ts *destination, const Ts *source) {
Ts min = experiment.GetUnderflow() + 1;
if (min > 0) min = 0;
Ts max = experiment.GetOverflow() - 1;
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++) {
if (experiment.GetBinning2x2())
TransferModuleAdjustMultipixelsBin2x2<Ts, float>(destination + experiment.GetPixel0OfModule(module_number),
source + module_number * RAW_MODULE_SIZE,
(experiment.IsUpsideDown() ? -1 : 1) * experiment.GetXPixelsNum(), min, max);
else
TransferModuleAdjustMultipixels<Ts, Ts>(destination + experiment.GetPixel0OfModule(module_number),
source + module_number * RAW_MODULE_SIZE,
(experiment.IsUpsideDown() ? -1 : 1) * experiment.GetXPixelsNum(), min, max);
}
}
template <class T> void LineConvtToRaw(T *destination, const T* source) {
for (int chip = 0; chip < 4; chip++) {
for (int i = 0; i < 256; i++)
destination[i+chip*256] = source[i+chip*258];
}
}
template <class T> void ConvertedToRawGeometry(const DiffractionExperiment& experiment, T *destination, const T* source) {
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++) {
for (size_t line = 0; line < RAW_MODULE_LINES; line++) {
LineConvtToRaw<T>(destination + module_number * RAW_MODULE_SIZE + RAW_MODULE_COLS * line,
source + experiment.GetPixel0OfModule(module_number) + (experiment.IsUpsideDown() ? -1 : 1) * experiment.GetXPixelsNum() * (line + ((line > 255) ? 2 : 0))
);
}
}
}
template <class T> void Bin2x2_or(T *destination, const T* source, size_t width, size_t height) {
for (int y = 0; y < height / 2; y++) {
for (int x = 0; x < width / 2; x++) {
T tmp[4];
tmp[0] = source[(y * 2) * width + (x * 2)];
tmp[1] = source[(y * 2 + 1) * width + (x * 2)];
tmp[2] = source[(y * 2) * width + (x * 2 + 1)];
tmp[3] = source[(y * 2 + 1) * width + (x * 2 + 1)];
destination[y * (width/2) + x] = tmp[0] | tmp[1] | tmp[2] | tmp[3];
}
}
}
template <class T, class Tint = int32_t>
void Bin2x2_sum(T *destination, const T* source, size_t width, size_t height, T underload, T overload) {
for (int y = 0; y < height / 2; y++) {
for (int x = 0; x < width / 2; x++)
destination[y * (width/2) + x] = Bin2x2_sum<T, Tint>(source[(y * 2) * width + (x * 2)],
source[(y * 2 + 1) * width + (x * 2)],
source[(y * 2) * width + (x * 2 + 1)],
source[(y * 2 + 1) * width + (x * 2 + 1)],
underload, overload);
}
}
#endif

13
common/SpotToSave.h Normal file
View File

@@ -0,0 +1,13 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_SPOTTOSAVE_H
#define JUNGFRAUJOCH_SPOTTOSAVE_H
struct SpotToSave {
float x;
float y;
float intensity;
};
#endif //JUNGFRAUJOCH_SPOTTOSAVE_H

88
common/StatusVector.h Normal file
View File

@@ -0,0 +1,88 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_STATUSVECTOR_H
#define JUNGFRAUJOCH_STATUSVECTOR_H
#include <vector>
#include <map>
#include <mutex>
#include <jfjoch.pb.h>
#include "JFJochException.h"
template <class T> class StatusVector {
mutable std::mutex m;
std::map<uint32_t, T > content;
uint32_t max_id = 0;
public:
void AddElement(uint32_t id, T val) {
std::unique_lock<std::mutex> ul(m);
content[id] = val;
if (id > max_id)
max_id = id;
}
[[nodiscard]] float Mean() const {
std::unique_lock<std::mutex> ul(m);
size_t count = content.size();
double sum = 0;
for (auto &[x,y]: content)
sum += y;
if (count > 0)
return static_cast<float>(sum / count);
else
return NAN;
}
[[nodiscard]] std::vector<float> GetStatus(int32_t bin_size) const {
std::unique_lock<std::mutex> ul(m);
if (bin_size <= 0)
throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds,
"Bin number must be greater than zero");
std::vector<float> ret;
if (!content.empty()) {
for (int bin = 0; bin < max_id / bin_size + ((max_id % bin_size > 0) ? 1 : 0); bin++) {
double sum = 0;
int64_t count = 0;
auto it_start = content.lower_bound(bin * bin_size);
auto it_end = content.upper_bound((bin + 1) * bin_size);
if ((it_start != content.end()) && (it_start->first < (bin + 1) * bin_size)) {
for (auto it = it_start; it != it_end; it++) {
sum += it->second;
count++;
}
}
if (count > 0)
ret.push_back(static_cast<float>(sum / static_cast<double>(count)));
else
ret.push_back(0.0);
}
}
return ret;
}
void GetPlot(JFJochProtoBuf::Plot& plot, int32_t bin_size) const {
// GetStatus has mutex, no need to lock again
auto status = GetStatus(bin_size);
if (status.size() == 1) {
plot.add_x(max_id / 2.0);
plot.add_y(status[0]);
} else if (!status.empty()) {
*plot.mutable_y() = {status.begin(), status.end()};
for (int i = 0; i < status.size(); i++)
plot.add_x(bin_size * (i + 0.5));
}
}
};
#endif //JUNGFRAUJOCH_STATUSVECTOR_H

126
common/StrongPixelSet.cpp Normal file
View File

@@ -0,0 +1,126 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <bitset>
#include "StrongPixelSet.h"
StrongPixelSet::StrongPixelSet(const DiffractionExperiment &experiment)
: xpixel(experiment.GetXPixelsNum()),
ypixel(experiment.GetYPixelsNum()),
strong_pixel_vector(experiment.GetPixelsNum(), false) {
}
void StrongPixelSet::AddStrongPixel(uint16_t col, uint16_t line, int32_t photons) {
auto key = strong_pixel_coord(col, line);
strong_pixel_map[key] = photons;
strong_pixel_vector.at(line * xpixel + col) = true;
}
inline void StrongPixelSet::AddNeighbor(DiffractionSpot &spot, uint16_t col, uint16_t line) {
if (strong_pixel_vector[line * xpixel + col]) {
uint64_t coord = strong_pixel_coord(col, line);
auto iter = strong_pixel_map.find(coord);
ExtendSpot(spot, iter);
}
}
// Creates a continuous spot
// strong pixels are loaded into dictionary (one dictionary per frame)
// and routine checks if neighboring pixels are also in dictionary (likely in log(N) time)
DiffractionSpot StrongPixelSet::BuildSpot(std::unordered_map<uint32_t, int32_t>::iterator &it) {
uint16_t col = col_from_strong_pixel(it->first);
uint16_t line = line_from_strong_pixel(it->first);
DiffractionSpot spot(col, line, it->second);
strong_pixel_vector[line * xpixel + col] = false;
strong_pixel_map.erase(it); // Remove strong pixel from the dictionary, so it is not processed again
if (col+1 < xpixel) {
AddNeighbor(spot, col + 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col + 1, line + 1);
if (line > 0)
AddNeighbor(spot, col + 1, line - 1);
}
if (col != 0) {
AddNeighbor(spot, col - 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col - 1, line + 1);
if (line > 0)
AddNeighbor(spot, col - 1, line - 1);
}
if (line < ypixel - 1)
AddNeighbor(spot, col, line+1);
if (line > 0)
AddNeighbor(spot, col, line-1);
return spot;
}
void StrongPixelSet::ExtendSpot(DiffractionSpot &spot, std::unordered_map<uint32_t, int32_t>::iterator &it) {
uint16_t col = col_from_strong_pixel(it->first);
uint16_t line = line_from_strong_pixel(it->first);
spot.AddPixel(col, line, it->second);
strong_pixel_vector[line * xpixel + col] = false;
strong_pixel_map.erase(it); // Remove strong pixel from the dictionary, so it is not processed again
if (col+1 < xpixel) {
AddNeighbor(spot, col + 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col + 1, line + 1);
if (line > 0)
AddNeighbor(spot, col + 1, line - 1);
}
if (col != 0) {
AddNeighbor(spot, col - 1, line);
if (line < ypixel - 1)
AddNeighbor(spot, col - 1, line + 1);
if (line > 0)
AddNeighbor(spot, col - 1, line - 1);
}
if (line < ypixel - 1)
AddNeighbor(spot, col, line+1);
if (line > 0)
AddNeighbor(spot, col, line-1);
}
void StrongPixelSet::FindSpots(const DiffractionExperiment &experiment, const JFJochProtoBuf::DataProcessingSettings &settings,
std::vector<DiffractionSpot> &spots) {
std::multimap<double, DiffractionSpot> spots_map;
while (!strong_pixel_map.empty()) {
auto iter = strong_pixel_map.begin();
DiffractionSpot spot = BuildSpot(iter);
double d = spot.GetResolution(experiment);
if ((spot.PixelCount() <= settings.max_pix_per_spot())
&& (spot.PixelCount() >= settings.min_pix_per_spot())
&& (!settings.has_low_resolution_limit() || (d <= settings.low_resolution_limit()))
&& (!settings.has_high_resolution_limit() || (d >= settings.high_resolution_limit())))
spots_map.insert(std::make_pair(-static_cast<float>(d), spot));
}
for (auto &[x, spot]: spots_map)
spots.push_back(spot);
if (experiment.GetMaxSpotCount() > 0)
spots.resize(std::min<size_t>(spots.size(), experiment.GetMaxSpotCount()));
}
size_t StrongPixelSet::Count() const {
return strong_pixel_map.size();
}
size_t StrongPixelSet::Common(const StrongPixelSet &set) const {
size_t ret = 0;
for (const auto& pixel: strong_pixel_map) {
if (set.strong_pixel_map.count(pixel.first) == 1)
ret++;
}
return ret;
}

42
common/StrongPixelSet.h Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_STRONGPIXELSET_H
#define JUNGFRAUJOCH_STRONGPIXELSET_H
#include <unordered_map>
#include <mutex>
#include "DiffractionExperiment.h"
#include "Coord.h"
#include "DiffractionSpot.h"
inline uint32_t strong_pixel_coord(uint16_t col, uint16_t line) {
return col + (static_cast<uint32_t>(line) << 16u);
}
inline uint16_t line_from_strong_pixel(uint32_t strong_pixel) {
return ((strong_pixel & 0xFFFF0000u) >> 16u);
}
inline uint16_t col_from_strong_pixel(uint32_t strong_pixel) {
return (strong_pixel & 0x0000FFFFu);
}
class StrongPixelSet {
std::unordered_map<uint32_t, int32_t> strong_pixel_map;
uint32_t xpixel;
uint32_t ypixel;
std::vector<bool> strong_pixel_vector;
void AddNeighbor(DiffractionSpot &spot, uint16_t col, uint16_t line);
DiffractionSpot BuildSpot(std::unordered_map<uint32_t, int32_t>::iterator &it_frames);
void ExtendSpot(DiffractionSpot &spot, std::unordered_map<uint32_t, int32_t>::iterator &it_frames);
public:
StrongPixelSet(const DiffractionExperiment& experiment);
size_t Count() const;
void AddStrongPixel(uint16_t col, uint16_t line, int32_t photons = 1);
void FindSpots(const DiffractionExperiment &experiment, const JFJochProtoBuf::DataProcessingSettings &settings, std::vector<DiffractionSpot> &spots);
size_t Common(const StrongPixelSet &set) const;
};
#endif //JUNGFRAUJOCH_STRONGPIXELSET_H

110
common/TestImagePusher.cpp Normal file
View File

@@ -0,0 +1,110 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "TestImagePusher.h"
#include "../tests/FPGAUnitTest.h"
#include "JFJochCompressor.h"
#include "../compression/JFJochDecompress.h"
#include "../frame_serialize/JFJochFrameDeserializer.h"
TestImagePusher::TestImagePusher(int64_t image_number) {
image_id = image_number;
}
void TestImagePusher::StartDataCollection(const StartMessage& message) {
std::unique_lock<std::mutex> ul(m);
if (is_running)
correct_sequence = false;
else
is_running = true;
}
void TestImagePusher::EndDataCollection(const EndMessage& message) {
std::unique_lock<std::mutex> ul(m);
if (!is_running)
correct_sequence = false;
else
is_running = false;
}
void TestImagePusher::SendDataInternal(const std::vector<uint8_t> &serialized_image, int64_t image_number) {
std::unique_lock<std::mutex> ul(m);
frame_counter++;
if (image_number == image_id) {
JFJochFrameDeserializer deserializer;
deserializer.Process(serialized_image);
auto image_array = deserializer.GetDataMessage();
receiver_generated_image.resize(image_array.image.size);
memcpy(receiver_generated_image.data(), image_array.image.data, image_array.image.size);
}
}
bool TestImagePusher::CheckSequence() const {
std::unique_lock<std::mutex> ul(m);
return correct_sequence;
}
const std::vector<uint8_t> &TestImagePusher::GetImage() const {
std::unique_lock<std::mutex> ul(m);
return receiver_generated_image;
}
size_t TestImagePusher::GetCounter() const {
std::unique_lock<std::mutex> ul(m);
return frame_counter;
}
bool TestImagePusher::CheckImage(const DiffractionExperiment &x, const std::vector<uint16_t> &raw_reference_image,
const JFCalibration &calibration,
Logger &logger) {
bool no_errors = true;
if (receiver_generated_image.empty()) {
logger.Error("Image empty");
no_errors = false;
} else {
std::vector<int16_t> decompressed_image;
// Image decompression
try {
JFJochDecompress(decompressed_image, x.GetCompressionAlgorithmEnum(),
receiver_generated_image, x.GetPixelsNum());
} catch (const JFJochException &e) {
logger.Error(e.what());
no_errors = false;
}
if (no_errors) {
// Check output
if (x.GetDetectorMode() == DetectorMode::Conversion) {
size_t storage_cell = 0;
if (x.GetStorageCellNumber() > 1)
storage_cell = image_id % x.GetStorageCellNumber();
double result = CheckConversionWithGeomTransform(x, calibration,
raw_reference_image.data(),
decompressed_image.data(),
storage_cell);
if (x.GetBinning2x2() && (result > 1.5)) {
logger.Error("Mean conversion error ({}) larger than threshold", result);
no_errors = false;
} else if (!x.GetBinning2x2() && (result > 0.5)) {
logger.Error("Mean conversion error ({}) larger than threshold", result);
no_errors = false;
} else
logger.Info("Mean conversion error: {}", result);
} else if (x.GetDetectorMode() == DetectorMode::Raw) {
if (memcmp(raw_reference_image.data(), decompressed_image.data(), sizeof(uint16_t) * x.GetPixelsNum()) !=
0) {
logger.Error("Raw data mismatch");
no_errors = false;
}
}
}
}
return no_errors;
}

37
common/TestImagePusher.h Normal file
View File

@@ -0,0 +1,37 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_TESTIMAGEPUSHER_H
#define JUNGFRAUJOCH_TESTIMAGEPUSHER_H
#include <mutex>
#include "ImagePusher.h"
#include "Logger.h"
#include "DiffractionExperiment.h"
#include "../jungfrau/JFCalibration.h"
#include "../jungfrau/JFModuleGainCalibration.h"
class TestImagePusher : public ImagePusher {
mutable std::mutex m;
std::vector<uint8_t> receiver_generated_image;
int64_t image_id;
bool correct_sequence = true;
bool is_running = false;
size_t frame_counter = 0;
void SendDataInternal(const std::vector<uint8_t> &serialized_image, int64_t image_number) override;
public:
explicit TestImagePusher(int64_t image_number);
void StartDataCollection(const StartMessage& message) override;
void EndDataCollection(const EndMessage& message) override;
bool CheckImage(const DiffractionExperiment &x,
const std::vector<uint16_t> &raw_reference_image,
const JFCalibration &calibration,
Logger &logger);
[[nodiscard]] bool CheckSequence() const;
[[nodiscard]] const std::vector<uint8_t> &GetImage() const;
[[nodiscard]] size_t GetCounter() const;
};
#endif //JUNGFRAUJOCH_TESTIMAGEPUSHER_H

109
common/ThreadSafeFIFO.h Normal file
View File

@@ -0,0 +1,109 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_THREADSAFEFIFO_H
#define JUNGFRAUJOCH_THREADSAFEFIFO_H
#include <queue>
#include <mutex>
#include <condition_variable>
#include <set>
template <class T> class ThreadSafeFIFO {
std::queue<T> queue;
std::condition_variable c_empty, c_full;
std::mutex m;
const size_t max_size;
public:
ThreadSafeFIFO(size_t in_max_size = UINT32_MAX) : max_size(in_max_size) {}
void Clear() {
std::unique_lock<std::mutex> ul(m);
queue = {};
}
bool Put(T val) {
std::unique_lock<std::mutex> ul(m);
if (queue.size() < max_size) {
queue.push(val);
c_empty.notify_one();
return true;
} else
return false;
};
void PutBlocking(T val) {
std::unique_lock<std::mutex> ul(m);
c_full.wait(ul, [&]{return queue.size() < max_size;});
queue.push(val);
c_empty.notify_one();
};
int Get(T &val) {
std::unique_lock<std::mutex> ul(m);
if (queue.empty())
return 0;
else {
val = queue.front();
queue.pop();
c_full.notify_one();
return 1;
}
}
T GetBlocking() {
std::unique_lock<std::mutex> ul(m);
c_empty.wait(ul, [&]{return !queue.empty();});
T tmp = queue.front();
queue.pop();
c_full.notify_one();
return tmp;
};
size_t Size() const {
return queue.size();
}
};
template <class T> class ThreadSafeSet {
std::set<T> set;
std::condition_variable c;
std::mutex m;
public:
void Clear() {
std::unique_lock<std::mutex> ul(m);
set = {};
}
void Put(T val) {
std::unique_lock<std::mutex> ul(m);
set.insert(val);
c.notify_one();
};
int Get(T &val) {
std::unique_lock<std::mutex> ul(m);
if (set.empty()) return 0;
else {
auto iter = set.begin();
val = *iter;
set.erase(iter);
return 1;
}
}
T GetBlocking() {
std::unique_lock<std::mutex> ul(m);
c.wait(ul, [&]{return !set.empty();});
auto iter = set.begin();
T tmp = *iter;
set.erase(iter);
return tmp;
};
size_t Size() const {
return set.size();
}
};
#endif //JUNGFRAUJOCH_THREADSAFEFIFO_H

16
common/UnitCell.h Normal file
View File

@@ -0,0 +1,16 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_UNITCELL_H
#define JUNGFRAUJOCH_UNITCELL_H
struct UnitCell {
float a;
float b;
float c;
float alpha;
float beta;
float gamma;
};
#endif //JUNGFRAUJOCH_UNITCELL_H

48
common/WriteTIFF.cpp Normal file
View File

@@ -0,0 +1,48 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "WriteTIFF.h"
#include "JFJochException.h"
#include <tiffio.h>
#include <tiffio.hxx>
#include <sstream>
void WriteTIFF(TIFF *tiff, void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed) {
if (tiff == nullptr)
throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFStreamOpen error");
TIFFSetField(tiff, TIFFTAG_IMAGEWIDTH, cols); // set the width of the image
TIFFSetField(tiff, TIFFTAG_IMAGELENGTH, lines); // set the height of the image
TIFFSetField(tiff, TIFFTAG_SAMPLESPERPIXEL, 1); // set number of channels per pixel
TIFFSetField(tiff, TIFFTAG_BITSPERSAMPLE, elem_size * 8); // set the size of the channels
TIFFSetField(tiff, TIFFTAG_COMPRESSION, COMPRESSION_LZW); // setc ompression to LZW
TIFFSetField(tiff, TIFFTAG_ROWSPERSTRIP, lines);
if (is_signed)
TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_INT);
else
TIFFSetField(tiff, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_UINT);
if (TIFFWriteEncodedStrip(tiff, 0, buff,cols * lines * elem_size) < 0)
throw JFJochException(JFJochExceptionCategory::TIFFGeneratorError, "TIFFWriteEncodedStrip error");
}
std::string WriteTIFFToString(void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed) {
std::stringstream os;
TIFF *tiff = TIFFStreamOpen("x", (std::ostream *) &os);
WriteTIFF(tiff, buff, cols, lines, elem_size, is_signed);
TIFFClose(tiff);
return os.str();
}
void WriteTIFFToFile(const std::string &filename, void *buff, size_t cols, size_t lines, size_t elem_size,
bool is_signed) {
TIFF *tiff = TIFFOpen(filename.c_str(), "w");
WriteTIFF(tiff, buff, cols, lines, elem_size, is_signed);
TIFFClose(tiff);
}

13
common/WriteTIFF.h Normal file
View File

@@ -0,0 +1,13 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_WRITETIFF_H
#define JUNGFRAUJOCH_WRITETIFF_H
#include <string>
std::string WriteTIFFToString(void *buff, size_t cols, size_t lines, size_t elem_size, bool is_signed = false);
void WriteTIFFToFile(const std::string &filename, void *buff, size_t cols, size_t lines, size_t elem_size,
bool is_signed = false);
#endif //JUNGFRAUJOCH_WRITETIFF_H

77
common/ZMQImagePusher.cpp Normal file
View File

@@ -0,0 +1,77 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "ZMQImagePusher.h"
#include "JFJochException.h"
ZMQImagePusher::ZMQImagePusher(ZMQContext &zmq_context, const std::vector<std::string> &addr,
int32_t send_buffer_high_watermark, int32_t send_buffer_size) {
if (addr.empty())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"No writer ZMQ address provided");
for (const auto &a : addr) {
auto s = std::make_unique<ZMQSocket>(zmq_context, ZMQSocketType::Push);
if (send_buffer_size > 0)
s->SendBufferSize(send_buffer_size);
if (send_buffer_high_watermark > 0)
s->SendWaterMark(send_buffer_high_watermark);
s->Bind(a);
sockets.push_back(std::move(s));
}
}
ZMQImagePusher::ZMQImagePusher(const std::vector<std::string> &addr,
int32_t send_buffer_high_watermark, int32_t send_buffer_size) {
if (addr.empty())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"No writer ZMQ address provided");
for (const auto &a : addr) {
auto c = std::make_unique<ZMQContext>();
auto s = std::make_unique<ZMQSocket>(*c, ZMQSocketType::Push);
if (send_buffer_size > 0)
s->SendBufferSize(send_buffer_size);
if (send_buffer_high_watermark > 0)
s->SendWaterMark(send_buffer_high_watermark);
s->Bind(a);
contexts.push_back(std::move(c));
sockets.push_back(std::move(s));
}
}
void ZMQImagePusher::SendDataInternal(const std::vector<uint8_t> &serialized_image, int64_t image_number) {
if (sockets.empty())
return;
auto socket_number = (image_number % file_count) % sockets.size();
sockets[socket_number]->Send(serialized_image.data(), serialized_image.size());
}
void ZMQImagePusher::StartDataCollection(const StartMessage& message) {
JFJochFrameSerializer serializer(80*1024*1024); // 80 MiB should be safe even for 16M
if (message.data_file_count < 1)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"File count cannot be zero or negative");
file_count = message.data_file_count;
serializer.SerializeSequenceStart(message);
auto &buffer = serializer.GetBuffer();
for (const auto &s: sockets)
s->Send(buffer.data(), buffer.size(), true);
}
void ZMQImagePusher::EndDataCollection(const EndMessage& message) {
JFJochFrameSerializer serializer(1024*1024);
EndMessage end_message = message;
for (const auto &s: sockets) {
serializer.SerializeSequenceEnd(end_message);
auto &buffer = serializer.GetBuffer();
s->Send(buffer.data(), buffer.size(), true);
end_message.write_master_file = false;
}
}

30
common/ZMQImagePusher.h Normal file
View File

@@ -0,0 +1,30 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_ZMQIMAGEPUSHER_H
#define JUNGFRAUJOCH_ZMQIMAGEPUSHER_H
#include <jfjoch.pb.h>
#include "ImagePusher.h"
#include "ThreadSafeFIFO.h"
#include "ZMQWrappers.h"
#include "DiffractionSpot.h"
#include "../frame_serialize/JFJochFrameSerializer.h"
class ZMQImagePusher : public ImagePusher {
std::vector<std::unique_ptr<ZMQContext>> contexts;
std::vector<std::unique_ptr<ZMQSocket>> sockets;
int64_t file_count = 1;
void SendDataInternal(const std::vector<uint8_t>& serialized_image, int64_t image_number) override;
public:
ZMQImagePusher(ZMQContext &context, const std::vector<std::string>& addr,
int32_t send_buffer_high_watermark = -1, int32_t send_buffer_size = -1);
// High performance implementation, where each socket has dedicated ZMQ context
explicit ZMQImagePusher(const std::vector<std::string>& addr,
int32_t send_buffer_high_watermark = -1, int32_t send_buffer_size = -1);
void StartDataCollection(const StartMessage& message) override;
void EndDataCollection(const EndMessage& message) override;
};
#endif //JUNGFRAUJOCH_ZMQIMAGEPUSHER_H

54
common/ZMQPreviewPublisher.cpp Normal file
View File

@@ -0,0 +1,54 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "ZMQPreviewPublisher.h"
#include "grpcToJson.h"
ZMQPreviewPublisher::ZMQPreviewPublisher(ZMQContext& context, const std::string& addr) :
socket(context, ZMQSocketType::Pub) {
socket.SendWaterMark(2).NoLinger();
socket.Bind(addr);
}
void ZMQPreviewPublisher::Start(const DiffractionExperiment &experiment, const JFCalibration &calibration) {
auto mask = calibration.CalculateNexusMask(experiment);
JFJochProtoBuf::PreviewFrame frame;
frame.set_image_number(-1);
frame.set_width(experiment.GetXPixelsNum());
frame.set_height(experiment.GetYPixelsNum());
frame.set_pixel_depth(4);
frame.set_data(mask.data(), experiment.GetPixelsNum() * sizeof(uint32_t));
socket.Send(grpcToJson(frame));
}
void ZMQPreviewPublisher::Stop(const DiffractionExperiment& experiment) {}
void ZMQPreviewPublisher::Publish(const DiffractionExperiment& experiment, const int16_t* image_data, uint32_t image_number) {
JFJochProtoBuf::PreviewFrame frame;
frame.set_image_number(image_number);
frame.set_total_images(experiment.GetImageNum());
frame.set_wavelength_a(experiment.GetWavelength_A());
frame.set_beam_x_pxl(experiment.GetBeamX_pxl());
frame.set_beam_y_pxl(experiment.GetBeamY_pxl());
frame.set_saturation_value(experiment.GetOverflow());
frame.set_file_prefix(experiment.GetFilePrefix());
frame.set_detector_distance_mm(experiment.GetDetectorDistance_mm());
frame.set_width(experiment.GetXPixelsNum());
frame.set_height(experiment.GetYPixelsNum());
frame.set_pixel_depth(2);
frame.set_data(image_data, experiment.GetPixelsNum() * sizeof(int16_t));
SetPreviewImage(frame);
socket.Send(grpcToJson(frame));
}
void ZMQPreviewPublisher::SetPreviewImage(const JFJochProtoBuf::PreviewFrame &frame) {
std::unique_lock<std::mutex> ul(frame_mutex);
saved_frame = frame;
}
JFJochProtoBuf::PreviewFrame ZMQPreviewPublisher::GetPreviewImage() const {
std::unique_lock<std::mutex> ul(frame_mutex);
return saved_frame;
}

29
common/ZMQPreviewPublisher.h Normal file
View File

@@ -0,0 +1,29 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_ZMQPREVIEWPUBLISHER_H
#define JUNGFRAUJOCH_ZMQPREVIEWPUBLISHER_H
#include <jfjoch.pb.h>
#include "ZMQWrappers.h"
#include "DiffractionExperiment.h"
#include "../jungfrau/JFCalibration.h"
class ZMQPreviewPublisher {
ZMQSocket socket;
mutable std::mutex frame_mutex;
JFJochProtoBuf::PreviewFrame saved_frame;
void SetPreviewImage(const JFJochProtoBuf::PreviewFrame &frame);
public:
ZMQPreviewPublisher(ZMQContext& context, const std::string& addr);
void Start(const DiffractionExperiment& experiment, const JFCalibration &calibration);
void Publish(const DiffractionExperiment& experiment, const int16_t* image_data, uint32_t image_number);
void Stop(const DiffractionExperiment& experiment);
JFJochProtoBuf::PreviewFrame GetPreviewImage() const;
};
#endif //JUNGFRAUJOCH_ZMQPREVIEWPUBLISHER_H

154
common/ZMQWrappers.cpp Normal file
View File

@@ -0,0 +1,154 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "ZMQWrappers.h"
#include <cerrno>
ZMQContext::ZMQContext() {
context = zmq_ctx_new();
}
ZMQContext &ZMQContext::NumThreads(int32_t threads) {
if (zmq_ctx_set(context, ZMQ_IO_THREADS, threads) != 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ,
"Cannot set number of I/O threads");
return *this;
}
ZMQContext::~ZMQContext() {
zmq_ctx_destroy(context);
}
void *ZMQContext::GetContext() const {
return context;
}
ZMQSocket::ZMQSocket(ZMQContext &context, ZMQSocketType in_socket_type) : socket_type(in_socket_type) {
socket = zmq_socket(context.GetContext(), static_cast<int>(socket_type));
if (socket == nullptr)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_socket failed");
}
void ZMQSocket::Bind(const std::string &addr) {
if (zmq_bind(socket, addr.c_str()) != 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_bind failed");
}
void ZMQSocket::Connect(const std::string &addr) {
if (zmq_connect(socket, addr.c_str()) != 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_connect failed");
}
void ZMQSocket::Disconnect(const std::string &addr) {
if (zmq_disconnect(socket, addr.c_str()) != 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_disconnect failed");
}
void ZMQSocket::Send() {
std::unique_lock<std::mutex> ul(m);
if (zmq_send(socket, nullptr, 0, 0) != 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_send() failed: " + std::string(std::strerror(errno)));
}
void ZMQSocket::Send(const void *buf, size_t buf_size, bool blocking, bool multipart) {
std::unique_lock<std::mutex> ul(m);
if (zmq_send(socket, buf, buf_size, (blocking ? 0 : ZMQ_DONTWAIT) | (multipart ? ZMQ_SNDMORE : 0)) != buf_size) {
if (errno != EAGAIN)
throw JFJochException(JFJochExceptionCategory::ZeroMQ,
"zmq_send(buf) failed: " + std::string(std::strerror(errno)));
}
}
void ZMQSocket::Send(const std::string &s, bool blocking, bool multipart) {
Send((void *) s.c_str(), s.size(), blocking, multipart);
}
void ZMQSocket::Send(const int32_t &value) {
std::unique_lock<std::mutex> ul(m);
if (zmq_send(socket, &value, sizeof(int32_t), 0) != sizeof(int32_t))
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_send(int) failed: " + std::string(std::strerror(errno)));
}
void ZMQSocket::Send(zmq_msg_t *msg) {
std::unique_lock<std::mutex> ul(m);
if (zmq_msg_send(msg, socket, 0) < 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_msg_send failed");
}
int64_t ZMQSocket::Receive(bool blocking) {
std::vector<uint8_t> msg;
return Receive(msg, blocking, true);
}
int64_t ZMQSocket::Receive(std::string &s, bool blocking) {
std::vector<uint8_t> v;
int64_t rc = Receive(v, blocking, true);
if (rc > 0)
s = std::string(v.begin(), v.end());
return rc;
}
void ZMQSocket::SetSocketOption(int32_t option_name, int32_t value) {
if (zmq_setsockopt(socket, option_name, &value, sizeof(value)) != 0)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Cannot set socket option");
}
ZMQSocket &ZMQSocket::ReceiveTimeout(std::chrono::milliseconds input) {
SetSocketOption(ZMQ_RCVTIMEO, input.count());
return *this;
}
ZMQSocket &ZMQSocket::NoReceiveTimeout() {
SetSocketOption(ZMQ_RCVTIMEO, -1);
return *this;
}
ZMQSocket &ZMQSocket::ReceiverBufferSize(int32_t bytes) {
SetSocketOption(ZMQ_RCVBUF, bytes);
return *this;
}
ZMQSocket &ZMQSocket::SendBufferSize(int32_t bytes) {
SetSocketOption(ZMQ_SNDBUF, bytes);
return *this;
}
ZMQSocket &ZMQSocket::SubscribeAll() {
if (socket_type != ZMQSocketType::Sub)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "subscribe only possible for Sub socket");
zmq_setsockopt(socket, ZMQ_SUBSCRIBE, nullptr, 0);
return *this;
}
ZMQSocket &ZMQSocket::Subscribe(const std::string &topic) {
if (socket_type != ZMQSocketType::Sub)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "subscribe only possible for Sub socket");
zmq_setsockopt(socket, ZMQ_SUBSCRIBE, topic.c_str(), topic.size());
return *this;
}
ZMQSocket::~ZMQSocket() {
zmq_close(socket);
}
ZMQSocket& ZMQSocket::ReceiveWaterMark(int32_t msgs) {
SetSocketOption(ZMQ_RCVHWM, msgs);
return *this;
}
ZMQSocket& ZMQSocket::SendWaterMark(int32_t msgs) {
SetSocketOption(ZMQ_SNDHWM, msgs);
return *this;
}
ZMQSocket &ZMQSocket::NoLinger() {
SetSocketOption(ZMQ_LINGER, 0);
return *this;
}
ZMQSocket &ZMQSocket::Conflate(bool input) {
SetSocketOption(ZMQ_CONFLATE, input ? 1 : 0);
return *this;
}

91
common/ZMQWrappers.h Normal file
View File

@@ -0,0 +1,91 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_ZMQWRAPPERS_H
#define JUNGFRAUJOCH_ZMQWRAPPERS_H
#include <vector>
#include <cstring>
#include <thread>
#include <mutex>
#include <zmq.h>
#include "JFJochException.h"
class ZMQContext {
void *context;
public:
ZMQContext();
ZMQContext& NumThreads(int32_t threads);
~ZMQContext();
void *GetContext() const;
};
enum class ZMQSocketType : int {Push = ZMQ_PUSH, Pull = ZMQ_PULL, Req = ZMQ_REQ, Rep = ZMQ_REP,
Pub = ZMQ_PUB, Sub = ZMQ_SUB};
class ZMQSocket {
std::mutex m;
ZMQSocketType socket_type;
void *socket;
void SetSocketOption(int32_t option_name, int32_t value);
public:
ZMQSocket(ZMQSocket &socket) = delete;
const ZMQSocket& operator=(ZMQSocket &socket) = delete;
ZMQSocket(ZMQContext &context, ZMQSocketType socket_type);
~ZMQSocket();
void Connect(const std::string& addr);
void Disconnect(const std::string& addr);
void Bind(const std::string& addr);
ZMQSocket &NoReceiveTimeout();
ZMQSocket &ReceiveTimeout(std::chrono::milliseconds input);
ZMQSocket &Subscribe(const std::string &topic);
ZMQSocket &SubscribeAll();
ZMQSocket &NoLinger();
ZMQSocket &Conflate(bool input);
ZMQSocket &SendBufferSize(int32_t bytes);
ZMQSocket &ReceiverBufferSize(int32_t bytes);
int64_t Receive(bool blocking = true);
int64_t Receive(std::string &j, bool blocking = true);
template <class T> int64_t Receive(std::vector<T> &vector, bool blocking = true, bool resize = true) {
std::unique_lock<std::mutex> ul(m);
zmq_msg_t zmq_msg;
zmq_msg_init(&zmq_msg);
int64_t msg_size = zmq_msg_recv(&zmq_msg, socket, blocking ? 0 : ZMQ_DONTWAIT);
if (msg_size < 0) {
if (errno == EAGAIN)
return -1;
if (errno == EINTR) // Timeout ?
return -1;
else
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "zmq_msg_recv failed "
+ std::string(strerror(errno)));
} else if (msg_size == 0) {
zmq_msg_close (&zmq_msg);
return 0;
} else if (resize) {
vector.resize(msg_size / sizeof(T) + ((msg_size % sizeof(T) != 0) ? 1 : 0));
} else {
zmq_msg_close (&zmq_msg);
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Receive buffer too small");
}
memcpy(vector.data(), zmq_msg_data(&zmq_msg), msg_size);
zmq_msg_close (&zmq_msg);
return msg_size;
}
void Send();
void Send(const void *buf, size_t buf_size, bool blocking = true, bool multipart = false);
template <class T> void Send(const std::vector<T> &buf) {
Send(buf.data(), buf.size() * sizeof(T));
}
void Send(const int32_t &value);
void Send(const std::string &s, bool blocking = true, bool multipart = false);
void Send(zmq_msg_t *msg);
ZMQSocket &SendWaterMark(int32_t msgs);
ZMQSocket &ReceiveWaterMark(int32_t msgs);
};
#endif //JUNGFRAUJOCH_ZMQWRAPPERS_H

25
common/grpcToJson.h Normal file
View File

@@ -0,0 +1,25 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_GRPCTOJSON_H
#define JUNGFRAUJOCH_GRPCTOJSON_H
#include <google/protobuf/util/json_util.h>
#include "../common/JFJochException.h"
inline std::string grpcToJson(const google::protobuf::Message &message) {
google::protobuf::util::JsonPrintOptions opts;
opts.always_print_primitive_fields = true;
opts.add_whitespace = true;
opts.always_print_enums_as_ints = false;
opts.preserve_proto_field_names = true;
std::string s;
auto status = google::protobuf::util::MessageToJsonString(message, &s, opts);
if (!status.ok())
throw JFJochException(JFJochExceptionCategory::JSON, "Error in generating JSON from ProtoBuf: " + status.message().ToString());
return s;
}
#endif //JUNGFRAUJOCH_GRPCTOJSON_H

25
common/jsonToGrpc.h Normal file
View File

@@ -0,0 +1,25 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JSONTOGRPC_H
#define JUNGFRAUJOCH_JSONTOGRPC_H
#include <google/protobuf/util/json_util.h>
#include "../common/JFJochException.h"
template<class T>
T jsonToGrpc(const std::string& json) {
T output;
google::protobuf::util::JsonParseOptions opts;
opts.case_insensitive_enum_parsing = false;
opts.ignore_unknown_fields = false;
auto status = google::protobuf::util::JsonStringToMessage(json, &output, opts);
if (!status.ok())
throw JFJochException(JFJochExceptionCategory::JSON, "Error in generating ProtoBuf from JSON: " + status.message().ToString());
return output;
}
#endif //JUNGFRAUJOCH_JSONTOGRPC_H

19
common/to_fixed.h Normal file
View File

@@ -0,0 +1,19 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_TO_FIXED_H
#define JUNGFRAUJOCH_TO_FIXED_H
#include <cstdint>
#include <cmath>
inline uint16_t to_fixed(double val, uint16_t fractional_bits) {
// If val is result of division by zero, only reasonable value of output is zero (otherwise number could be interpreted improperly)
uint32_t int_val = std::isfinite(val) ? (std::lround(val * (1<<fractional_bits))) : 0;
// It is unlikely (but not impossible), that gain value will be lower than the smallest possible
// Then reciprocal of gain could be more than allowed by data format. Protection is added for this condition
if (int_val > UINT16_MAX) int_val = UINT16_MAX;
return int_val;
}
#endif //JUNGFRAUJOCH_TO_FIXED_H

18
compression/CMakeLists.txt Normal file
View File

@@ -0,0 +1,18 @@
ADD_LIBRARY(Compression STATIC
lz4/lz4.c
lz4/lz4.h
bitshuffle/bitshuffle.c
bitshuffle/bitshuffle_core.c
bitshuffle/iochain.c
JFJochZstdCompressor.cpp
JFJochZstdCompressor.h
JFJochCompressor.cpp
JFJochCompressor.h
JFJochDecompress.h
MaxCompressedSize.cpp
MaxCompressedSize.h)
TARGET_COMPILE_DEFINITIONS(Compression PUBLIC -DZSTD_SUPPORT -DUSE_ZSTD)
TARGET_LINK_LIBRARIES(Compression libzstd_static)
TARGET_INCLUDE_DIRECTORIES(Compression PUBLIC . zstd/lib)
ADD_SUBDIRECTORY(zstd/build/cmake)

9
compression/CompressionAlgorithmEnum.h Normal file
View File

@@ -0,0 +1,9 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_COMPRESSIONALGORITHMENUM_H
#define JUNGFRAUJOCH_COMPRESSIONALGORITHMENUM_H
enum class CompressionAlgorithm {NO_COMPRESSION, BSHUF_LZ4, BSHUF_ZSTD, BSHUF_ZSTD_RLE};
#endif //JUNGFRAUJOCH_COMPRESSIONALGORITHMENUM_H

95
compression/JFJochCompressor.cpp Normal file
View File

@@ -0,0 +1,95 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "JFJochCompressor.h"
#include <stdexcept>
#include <cstring>
#include <bitshuffle/bitshuffle_internals.h>
#include <zstd.h>
#include <lz4/lz4.h>
#include "../common/JFJochException.h"
extern "C" {
void bshuf_write_uint64_BE(void* buf, uint64_t num);
}
JFJochBitShuffleCompressor::JFJochBitShuffleCompressor(CompressionAlgorithm in_algorithm) {
algorithm = in_algorithm;
}
size_t JFJochBitShuffleCompressor::CompressBlock(char *dest, const char *source, size_t nelements, size_t elem_size) {
// Assert nelements < block_size
const char *src_ptr;
int64_t bshuf_ret = bshuf_trans_bit_elem(source, tmp_space.data(), nelements, elem_size);
if (bshuf_ret < 0)
throw JFJochException(JFJochExceptionCategory::Compression, "bshuf_trans_bit_elem error");
src_ptr = tmp_space.data();
size_t compressed_size;
size_t src_size = nelements * elem_size;
switch (algorithm) {
case CompressionAlgorithm::BSHUF_LZ4:
compressed_size = LZ4_compress_default(src_ptr, dest + 4, src_size, LZ4_compressBound(src_size));
break;
case CompressionAlgorithm::BSHUF_ZSTD:
compressed_size = ZSTD_compress(dest + 4, ZSTD_compressBound(src_size), src_ptr, src_size, 0);
if (ZSTD_isError(compressed_size))
throw(JFJochException(JFJochExceptionCategory::Compression, ZSTD_getErrorName(compressed_size)));
break;
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
try {
compressed_size = zstd_compressor.Compress(((uint8_t *) dest) + 4, (uint64_t *) src_ptr,
src_size, src_size);
} catch (const std::runtime_error &e) {
throw JFJochException(JFJochExceptionCategory::ZSTDCompressionError, e.what());
}
break;
default:
throw JFJochException(JFJochExceptionCategory::Compression, "Algorithm not supported");
}
bshuf_write_uint32_BE(dest, compressed_size);
return compressed_size + 4;
}
size_t JFJochBitShuffleCompressor::Compress(char *dest, const char *source, size_t nelements, size_t elem_size) {
static_assert(DefaultBlockSize % BSHUF_BLOCKED_MULT == 0, "Block size must be multiple of 8");
if (algorithm == CompressionAlgorithm::NO_COMPRESSION) {
// Trivial case if no compression - copy content
memcpy(dest, source, nelements * elem_size);
return nelements * elem_size;
}
bshuf_write_uint64_BE(dest, nelements * elem_size);
bshuf_write_uint32_BE(dest + 8, DefaultBlockSize * elem_size);
if (tmp_space.size() < DefaultBlockSize * elem_size)
tmp_space.resize(DefaultBlockSize * elem_size);
size_t num_full_blocks = nelements / DefaultBlockSize;
size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize;
size_t compressed_size = 12;
for (int i = 0; i < num_full_blocks; i++)
compressed_size += CompressBlock(dest + compressed_size,
source + i * DefaultBlockSize * elem_size, DefaultBlockSize, elem_size);
size_t last_block_size = reminder_size - reminder_size % BSHUF_BLOCKED_MULT;
if (last_block_size > 0)
compressed_size += CompressBlock(dest + compressed_size,
source + num_full_blocks * DefaultBlockSize * elem_size, last_block_size, elem_size);
size_t leftover_bytes = (reminder_size % BSHUF_BLOCKED_MULT) * elem_size;
if (leftover_bytes > 0) {
memcpy(dest + compressed_size, source + (num_full_blocks * DefaultBlockSize + last_block_size) * elem_size, leftover_bytes);
compressed_size += leftover_bytes;
}
return compressed_size;
}

40
compression/JFJochCompressor.h Normal file
View File

@@ -0,0 +1,40 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JFJOCHCOMPRESSOR_H
#define JUNGFRAUJOCH_JFJOCHCOMPRESSOR_H
#include <bitshuffle/bitshuffle.h>
#include <vector>
#include <cstdint>
#include <cstddef>
#include "CompressionAlgorithmEnum.h"
#include "JFJochZstdCompressor.h"
class JFJochBitShuffleCompressor {
JFJochZstdCompressor zstd_compressor;
CompressionAlgorithm algorithm;
std::vector<char> tmp_space;
size_t CompressBlock(char *dest, const char * source, size_t nelements, size_t elem_size);
public:
constexpr static const size_t DefaultBlockSize = 4096;
JFJochBitShuffleCompressor(CompressionAlgorithm algorithm);
template<class T>
size_t Compress(void *dest, const std::vector<T> &src) {
return Compress((char *) dest, (char *) src.data(), src.size(), sizeof(T));
};
size_t Compress(char *dest, const char* source, size_t nelements, size_t elem_size);
};
template <class T> std::vector<T> bitshuffle(const std::vector<T> &input, size_t block_size) {
std::vector<T> ret(input.size() * sizeof(T));
bshuf_bitshuffle(input.data(), ret.data(), input.size(), sizeof(T), block_size);
return ret;
}
#endif //JUNGFRAUJOCH_JFJOCHCOMPRESSOR_H

60
compression/JFJochDecompress.h Normal file
View File

@@ -0,0 +1,60 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JFJOCHDECOMPRESS_H
#define JUNGFRAUJOCH_JFJOCHDECOMPRESS_H
#include <vector>
#include <cstring>
#include <bitshuffle/bitshuffle.h>
#include <bitshuffle/bitshuffle_internals.h>
#include "../compression/CompressionAlgorithmEnum.h"
#include "../common/JFJochException.h"
extern "C" {
uint64_t bshuf_read_uint64_BE(void* buf);
};
template <class Td, class Ts>
void JFJochDecompress(std::vector<Td> &output, CompressionAlgorithm algorithm, std::vector<Ts> source_v,
size_t nelements) {
size_t elem_size = sizeof(Td);
output.resize(nelements * elem_size);
size_t source_size = source_v.size() * sizeof(Ts);
auto source = (uint8_t *) source_v.data();
size_t block_size;
if (algorithm != CompressionAlgorithm::NO_COMPRESSION) {
if (bshuf_read_uint64_BE(source) != nelements * elem_size)
throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size");
auto tmp = bshuf_read_uint32_BE(source + 8);
block_size = tmp / elem_size;
}
switch (algorithm) {
case CompressionAlgorithm::NO_COMPRESSION:
if (source_size != nelements * elem_size)
throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size");
memcpy(output.data(), source, source_size);
break;
case CompressionAlgorithm::BSHUF_LZ4:
if (bshuf_decompress_lz4(source + 12, output.data(), nelements,
elem_size, block_size) != source_size - 12)
throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error");
break;
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
case CompressionAlgorithm::BSHUF_ZSTD:
if (bshuf_decompress_zstd(source + 12, output.data(), nelements,
elem_size, block_size) != source_size - 12)
throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error");
break;
default:
throw JFJochException(JFJochExceptionCategory::Compression, "Not implemented algorithm");
}
}
#endif //JUNGFRAUJOCH_JFJOCHDECOMPRESS_H

197
compression/JFJochZstdCompressor.cpp Normal file
View File

@@ -0,0 +1,197 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "JFJochZstdCompressor.h"
#include <map>
#include <zstd.h>
#include <lz4/lz4.h>
#include <common/mem.h>
#include <stdexcept>
enum class CompressorState {RLE_0, RLE_FF, RAW};
#define BLOCK_RAW 0
#define BLOCK_RLE 1
#define BLOCK_COMP 2
#define LITERAL_COMPRESSED 2
#define LITERAL_TREELESS 3
JFJochZstdCompressor::JFJochZstdCompressor() {
unsigned count[256];
for (int i = 0; i < 256; i++) count[i] = i;
}
size_t JFJochZstdCompressor::RawBlock(uint8_t *dst, const void *src, uint32_t src_size, bool last) {
size_t ret_value = 0;
int32_t bytes_left = src_size;
auto src_u8 = (uint8_t *)src;
while (bytes_left > 0) {
int32_t bytes_to_write = std::min(bytes_left,ZSTD_BLOCKSIZE_MAX);
bytes_left -= bytes_to_write;
uint32_t block_header = (bytes_to_write << 3) + (BLOCK_RAW<<1) + (((last && (bytes_left == 0)) ? 1 : 0));
MEM_writeLE24(dst, block_header);
memcpy(dst + 3, src_u8, bytes_to_write);
src_u8 += bytes_to_write;
ret_value += 3 + bytes_to_write;
dst += 3 + bytes_to_write;
}
return ret_value;
}
size_t JFJochZstdCompressor::RLEBlock(uint8_t *dst, uint8_t src, uint32_t src_size, bool last) {
size_t ret_value = 0;
int32_t bytes_left = src_size;
while (bytes_left > 0) {
int32_t bytes_to_write = std::min(bytes_left, ZSTD_BLOCKSIZE_MAX);
bytes_left -= bytes_to_write;
uint32_t block_header = (bytes_to_write << 3) + (BLOCK_RLE<<1) + ((last && (bytes_left == 0))? 1 : 0);
uint32_t block = block_header + (src<<24); // Little Endian!
MEM_writeLE32(dst + ret_value, block);
ret_value += 4;
}
return ret_value;
}
size_t JFJochZstdCompressor::DataBlock(uint8_t *dst, const void *src, uint32_t src_size, bool last) {
return RawBlock(dst, src, src_size, last);
}
size_t JFJochZstdCompressor::BlockStateMachine(uint8_t *dst, const uint64_t *src, size_t frame_size64) {
size_t dst_size = 0;
CompressorState state;
size_t seq_len = 8;
if (src[0] == 0x0) state = CompressorState::RLE_0;
else if (src[0] == UINT64_MAX) state = CompressorState::RLE_FF;
else state = CompressorState::RAW;
for (int i = 1; i < frame_size64; i++) {
switch (state) {
case CompressorState::RLE_0:
if (src[i] == 0x0) seq_len += 8;
else if ((src[i] & 0x00000000000000FF) != 0) {
dst_size += RLEBlock(dst + dst_size, 0x0, seq_len, false);
state = CompressorState::RAW;
seq_len = 8;
} else if (src[i] == UINT64_MAX) {
dst_size += RLEBlock(dst + dst_size, 0x0, seq_len, false);
state = CompressorState::RLE_FF;
seq_len = 8;
} else {
int delta1 = 1;
if ((src[i] & 0x00FFFFFFFFFFFFFF) == 0) delta1 = 7;
else if ((src[i] & 0x0000FFFFFFFFFFFF) == 0) delta1 = 6;
else if ((src[i] & 0x000000FFFFFFFFFF) == 0) delta1 = 5;
else if ((src[i] & 0x00000000FFFFFFFF) == 0) delta1 = 4;
else if ((src[i] & 0x0000000000FFFFFF) == 0) delta1 = 3;
else if ((src[i] & 0x000000000000FFFF) == 0) delta1 = 2;
dst_size += RLEBlock(dst + dst_size, 0x0, seq_len + delta1, false);
state = CompressorState::RAW;
seq_len = 8 - delta1;
}
break;
case CompressorState::RLE_FF:
if (src[i] == UINT64_MAX) seq_len += 8;
else {
dst_size += RLEBlock(dst + dst_size, 0xFFu, seq_len, false);
seq_len = 8;
if (src[i] == 0x0u)
state = CompressorState::RLE_0;
else
state = CompressorState::RAW;
}
break;
case CompressorState::RAW:
if (src[i] == 0x0u) {
dst_size += DataBlock(dst + dst_size, (uint8_t *) (src + i) - seq_len,
seq_len, false);
state = CompressorState::RLE_0;
seq_len = 8;
} else if ((i < frame_size64 - 1) && (src[i+1] == 0x0u) && ((src[i] & 0xFF00000000000000) == 0)) {
int delta2 = 1;
if ((src[i] & 0xFFFFFFFFFFFFFF00) == 0) delta2 = 7;
else if ((src[i] & 0xFFFFFFFFFFFF0000) == 0) delta2 = 6;
else if ((src[i] & 0xFFFFFFFFFF000000) == 0) delta2 = 5;
else if ((src[i] & 0xFFFFFFFF00000000) == 0) delta2 = 4;
else if ((src[i] & 0xFFFFFF0000000000) == 0) delta2 = 3;
else if ((src[i] & 0xFFFF000000000000) == 0) delta2 = 2;
dst_size += DataBlock(dst + dst_size, (uint8_t *) (src + i) - seq_len,
seq_len + 8 - delta2, false);
state = CompressorState::RLE_0;
seq_len = delta2;
} else if (src[i] == UINT64_MAX) {
dst_size += DataBlock(dst + dst_size, (uint8_t *) (src + i) - seq_len,
seq_len, false);
state = CompressorState::RLE_FF;
seq_len = 8;
} else seq_len += 8;
break;
}
}
if (state == CompressorState::RLE_0)
dst_size += RLEBlock(dst + dst_size, 0x0 , seq_len, true);
else if (state == CompressorState::RLE_FF)
dst_size += RLEBlock(dst + dst_size, 0xFFu, seq_len, true);
else
dst_size += DataBlock(dst + dst_size, (uint8_t *) (src + frame_size64) - seq_len, seq_len, true);
return dst_size;
}
size_t JFJochZstdCompressor::CompressFrame(uint8_t *dst, const uint64_t *src, size_t frame_size64) {
//if (frame_size64 > 1000 * 1000)
// throw SLSException(SLSExceptionCategory::ZSTDCompressionError,
// "ZSTD frame cannot be larger than 8 MB");
// Assumptions:
// One frame = One module = 512*1024*2 (2^20 == 1048576) - 514*1030*4 (2117680 < 2 ^22)
// --> frame size needs 32-bit
size_t dst_size = 0;
// Magic number
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
dst_size += 4;
// Frame header descriptor
uint8_t frame_header_descriptor = (1<<7) + (1<<5);
dst[dst_size] = frame_header_descriptor;
dst_size += 1;
// Frame size
MEM_writeLE32(dst + dst_size, frame_size64*8);
dst_size += 4;
dst_size += BlockStateMachine(dst + dst_size, src, frame_size64);
return dst_size;
}
size_t JFJochZstdCompressor::Compress(uint8_t *dst, const uint64_t *src, size_t src_size, size_t frame_size) {
size_t dst_size = 0;
if (frame_size % 8 != 0)
throw std::runtime_error("ZSTD frame has to have size multiple of 8");
if (src_size % frame_size != 0)
throw std::runtime_error("ZSTD source has to be multiple of frame");
size_t nframes = src_size / frame_size;
size_t frame_size64 = frame_size / 8;
for (int frame = 0; frame < nframes; frame++)
dst_size += CompressFrame(dst + dst_size, src + frame * frame_size64, frame_size64);
return dst_size;
}

22
compression/JFJochZstdCompressor.h Normal file
View File

@@ -0,0 +1,22 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_JFJOCHZSTDCOMPRESSOR_H
#define JUNGFRAUJOCH_JFJOCHZSTDCOMPRESSOR_H
#include <cstdint>
#include <cstddef>
class JFJochZstdCompressor {
size_t BlockStateMachine(uint8_t *dst, const uint64_t *src, size_t frame_size64);
size_t CompressFrame(uint8_t *dst, const uint64_t *src, size_t frame_size64);
size_t DataBlock(uint8_t *dst, const void *src, uint32_t src_size, bool last);
public:
JFJochZstdCompressor();
static size_t RawBlock(uint8_t *dst, const void *src, uint32_t src_size, bool last);
static size_t RLEBlock(uint8_t *dst, uint8_t src, uint32_t src_size, bool last);
size_t Compress(uint8_t *dst, const uint64_t *src, size_t src_size, size_t frame_sizes);
};
#endif //JUNGFRAUJOCH_JFJOCHZSTDCOMPRESSOR_H

20
compression/MaxCompressedSize.cpp Normal file
View File

@@ -0,0 +1,20 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <bitshuffle/bitshuffle.h>
#include "JFJochCompressor.h"
#include "MaxCompressedSize.h"
int64_t MaxCompressedSize(CompressionAlgorithm algorithm, int64_t pixels_number, uint16_t pixel_depth) {
switch (algorithm) {
case CompressionAlgorithm::BSHUF_LZ4:
return bshuf_compress_lz4_bound(pixels_number, pixel_depth, JFJochBitShuffleCompressor::DefaultBlockSize) + 12;
case CompressionAlgorithm::BSHUF_ZSTD:
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
return bshuf_compress_zstd_bound(pixels_number, pixel_depth,JFJochBitShuffleCompressor::DefaultBlockSize) + 12;
default:
return pixels_number * pixel_depth;
}
}

12
compression/MaxCompressedSize.h Normal file
View File

@@ -0,0 +1,12 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_MAXCOMPRESSEDSIZE_H
#define JUNGFRAUJOCH_MAXCOMPRESSEDSIZE_H
#include <cstdint>
#include "CompressionAlgorithmEnum.h"
int64_t MaxCompressedSize(CompressionAlgorithm algorithm, int64_t pixels_number, uint16_t pixel_depth);
#endif //JUNGFRAUJOCH_MAXCOMPRESSEDSIZE_H

21
compression/bitshuffle/LICENSE Normal file
View File

@@ -0,0 +1,21 @@
Bitshuffle - Filter for improving compression of typed binary data.
Copyright (c) 2014 Kiyoshi Masui (kiyo@physics.ubc.ca)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

279
compression/bitshuffle/bitshuffle.c Normal file
View File

@@ -0,0 +1,279 @@
/*
* Bitshuffle - Filter for improving compression of typed binary data.
*
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*/
#include "bitshuffle.h"
#include "bitshuffle_core.h"
#include "bitshuffle_internals.h"
#include "../lz4/lz4.h"
#ifdef ZSTD_SUPPORT
#include "zstd.h"
#endif
#include <stdio.h>
#include <string.h>
// Macros.
#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) { \
free(buf); return count - 1000; }
/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \
const size_t size, const size_t elem_size, const int option) {
int64_t nbytes, count;
void *tmp_buf_bshuf;
void *tmp_buf_lz4;
size_t this_iter;
const void *in;
void *out;
tmp_buf_bshuf = malloc(size * elem_size);
if (tmp_buf_bshuf == NULL) return -1;
int dst_capacity = LZ4_compressBound(size * elem_size);
tmp_buf_lz4 = malloc(dst_capacity);
if (tmp_buf_lz4 == NULL){
free(tmp_buf_bshuf);
return -1;
}
in = ioc_get_in(C_ptr, &this_iter);
ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size));
count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size);
if (count < 0) {
free(tmp_buf_lz4);
free(tmp_buf_bshuf);
return count;
}
nbytes = LZ4_compress_default((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size, dst_capacity);
free(tmp_buf_bshuf);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4);
out = ioc_get_out(C_ptr, &this_iter);
ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4));
bshuf_write_uint32_BE(out, nbytes);
memcpy((char *) out + 4, tmp_buf_lz4, nbytes);
free(tmp_buf_lz4);
return nbytes + 4;
}
/* Decompress and bitunshuffle a single block. */
int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int option) {
int64_t nbytes, count;
void *out, *tmp_buf;
const void *in;
size_t this_iter;
int32_t nbytes_from_header;
in = ioc_get_in(C_ptr, &this_iter);
nbytes_from_header = bshuf_read_uint32_BE(in);
ioc_set_next_in(C_ptr, &this_iter,
(void*) ((char*) in + nbytes_from_header + 4));
out = ioc_get_out(C_ptr, &this_iter);
ioc_set_next_out(C_ptr, &this_iter,
(void *) ((char *) out + size * elem_size));
tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
nbytes = LZ4_decompress_safe((const char*) in + 4, (char *) tmp_buf, nbytes_from_header,
size * elem_size);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
if (nbytes != size * elem_size) {
free(tmp_buf);
return -91;
}
nbytes = nbytes_from_header;
count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size);
CHECK_ERR_FREE(count, tmp_buf);
nbytes += 4;
free(tmp_buf);
return nbytes;
}
#ifdef ZSTD_SUPPORT
/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_zstd_block(ioc_chain *C_ptr, \
const size_t size, const size_t elem_size, const int comp_lvl) {
int64_t nbytes, count;
void *tmp_buf_bshuf;
void *tmp_buf_zstd;
size_t this_iter;
const void *in;
void *out;
tmp_buf_bshuf = malloc(size * elem_size);
if (tmp_buf_bshuf == NULL) return -1;
size_t tmp_buf_zstd_size = ZSTD_compressBound(size * elem_size);
tmp_buf_zstd = malloc(tmp_buf_zstd_size);
if (tmp_buf_zstd == NULL){
free(tmp_buf_bshuf);
return -1;
}
in = ioc_get_in(C_ptr, &this_iter);
ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size));
count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size);
if (count < 0) {
free(tmp_buf_zstd);
free(tmp_buf_bshuf);
return count;
}
nbytes = ZSTD_compress(tmp_buf_zstd, tmp_buf_zstd_size, (const void*)tmp_buf_bshuf, size * elem_size, comp_lvl);
free(tmp_buf_bshuf);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf_zstd);
out = ioc_get_out(C_ptr, &this_iter);
ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4));
bshuf_write_uint32_BE(out, nbytes);
memcpy((char *) out + 4, tmp_buf_zstd, nbytes);
free(tmp_buf_zstd);
return nbytes + 4;
}
/* Decompress and bitunshuffle a single block. */
int64_t bshuf_decompress_zstd_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int option) {
int64_t nbytes, count;
void *out, *tmp_buf;
const void *in;
size_t this_iter;
int32_t nbytes_from_header;
in = ioc_get_in(C_ptr, &this_iter);
nbytes_from_header = bshuf_read_uint32_BE(in);
ioc_set_next_in(C_ptr, &this_iter,
(void*) ((char*) in + nbytes_from_header + 4));
out = ioc_get_out(C_ptr, &this_iter);
ioc_set_next_out(C_ptr, &this_iter,
(void *) ((char *) out + size * elem_size));
tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
nbytes = ZSTD_decompress(tmp_buf, size * elem_size, in + 4, nbytes_from_header);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
if (nbytes != size * elem_size) {
free(tmp_buf);
return -91;
}
nbytes = nbytes_from_header;
count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size);
CHECK_ERR_FREE(count, tmp_buf);
nbytes += 4;
free(tmp_buf);
return nbytes;
}
#endif // ZSTD_SUPPORT
/* ---- Public functions ----
*
* See header file for description and usage.
*
*/
size_t bshuf_compress_lz4_bound(const size_t size,
const size_t elem_size, size_t block_size) {
size_t bound, leftover;
if (block_size == 0) {
block_size = bshuf_default_block_size(elem_size);
}
if (block_size % BSHUF_BLOCKED_MULT) return -81;
// Note that each block gets a 4 byte header.
// Size of full blocks.
bound = (LZ4_compressBound(block_size * elem_size) + 4) * (size / block_size);
// Size of partial blocks, if any.
leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
if (leftover) bound += LZ4_compressBound(leftover * elem_size) + 4;
// Size of uncompressed data not fitting into any blocks.
bound += (size % BSHUF_BLOCKED_MULT) * elem_size;
return bound;
}
int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size) {
return bshuf_blocked_wrap_fun(&bshuf_compress_lz4_block, in, out, size,
elem_size, block_size, 0/*option*/);
}
int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size) {
return bshuf_blocked_wrap_fun(&bshuf_decompress_lz4_block, in, out, size,
elem_size, block_size, 0/*option*/);
}
#ifdef ZSTD_SUPPORT
size_t bshuf_compress_zstd_bound(const size_t size,
const size_t elem_size, size_t block_size) {
size_t bound, leftover;
if (block_size == 0) {
block_size = bshuf_default_block_size(elem_size);
}
if (block_size % BSHUF_BLOCKED_MULT) return -81;
// Note that each block gets a 4 byte header.
// Size of full blocks.
bound = (ZSTD_compressBound(block_size * elem_size) + 4) * (size / block_size);
// Size of partial blocks, if any.
leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
if (leftover) bound += ZSTD_compressBound(leftover * elem_size) + 4;
// Size of uncompressed data not fitting into any blocks.
bound += (size % BSHUF_BLOCKED_MULT) * elem_size;
return bound;
}
int64_t bshuf_compress_zstd(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size, const int comp_lvl) {
return bshuf_blocked_wrap_fun(&bshuf_compress_zstd_block, in, out, size,
elem_size, block_size, comp_lvl);
}
int64_t bshuf_decompress_zstd(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size) {
return bshuf_blocked_wrap_fun(&bshuf_decompress_zstd_block, in, out, size,
elem_size, block_size, 0/*option*/);
}
#endif // ZSTD_SUPPORT

215
compression/bitshuffle/bitshuffle.h Normal file
View File

@@ -0,0 +1,215 @@
/*
* Bitshuffle - Filter for improving compression of typed binary data.
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*
* Header File
*
* Worker routines return an int64_t which is the number of bytes processed
* if positive or an error code if negative.
*
* Error codes:
* -1 : Failed to allocate memory.
* -11 : Missing SSE.
* -12 : Missing AVX.
* -80 : Input size not a multiple of 8.
* -81 : block_size not multiple of 8.
* -91 : Decompression error, wrong number of bytes processed.
* -1YYY : Error internal to compression routine with error code -YYY.
*/
#ifndef BITSHUFFLE_H
#define BITSHUFFLE_H
#include <stdlib.h>
#include "bitshuffle_core.h"
#ifdef __cplusplus
extern "C" {
#endif
/*
* ---- LZ4 Interface ----
*/
/* ---- bshuf_compress_lz4_bound ----
*
* Bound on size of data compressed with *bshuf_compress_lz4*.
*
* Parameters
* ----------
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Process in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* Bound on compressed data size.
*
*/
size_t bshuf_compress_lz4_bound(const size_t size,
const size_t elem_size, size_t block_size);
/* ---- bshuf_compress_lz4 ----
*
* Bitshuffled and compress the data using LZ4.
*
* Transpose within elements, in blocks of data of *block_size* elements then
* compress the blocks using LZ4. In the output buffer, each block is prefixed
* by a 4 byte integer giving the compressed size of that block.
*
* Output buffer must be large enough to hold the compressed data. This could
* be in principle substantially larger than the input buffer. Use the routine
* *bshuf_compress_lz4_bound* to get an upper limit.
*
* Parameters
* ----------
* in : input buffer, must be of size * elem_size bytes
* out : output buffer, must be large enough to hold data.
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Process in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* number of bytes used in output buffer, negative error-code if failed.
*
*/
int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, const size_t
elem_size, size_t block_size);
/* ---- bshuf_decompress_lz4 ----
*
* Undo compression and bitshuffling.
*
* Decompress data then un-bitshuffle it in blocks of *block_size* elements.
*
* To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
* must patch the parameters used to compress the data.
*
* NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function
* LZ4_decompress_fast from LZ4, which does not protect against maliciously
* formed datasets. By modifying the compressed data, this function could be
* coerced into leaving the boundaries of the input buffer.
*
* Parameters
* ----------
* in : input buffer
* out : output buffer, must be of size * elem_size bytes
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Process in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* number of bytes consumed in *input* buffer, negative error-code if failed.
*
*/
int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size);
/*
* ---- ZSTD Interface ----
*/
#ifdef ZSTD_SUPPORT
/* ---- bshuf_compress_zstd_bound ----
*
* Bound on size of data compressed with *bshuf_compress_zstd*.
*
* Parameters
* ----------
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Process in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* Bound on compressed data size.
*
*/
size_t bshuf_compress_zstd_bound(const size_t size,
const size_t elem_size, size_t block_size);
/* ---- bshuf_compress_zstd ----
*
* Bitshuffled and compress the data using zstd.
*
* Transpose within elements, in blocks of data of *block_size* elements then
* compress the blocks using ZSTD. In the output buffer, each block is prefixed
* by a 4 byte integer giving the compressed size of that block.
*
* Output buffer must be large enough to hold the compressed data. This could
* be in principle substantially larger than the input buffer. Use the routine
* *bshuf_compress_zstd_bound* to get an upper limit.
*
* Parameters
* ----------
* in : input buffer, must be of size * elem_size bytes
* out : output buffer, must be large enough to hold data.
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Process in blocks of this many elements. Pass 0 to
* select automatically (recommended).
* comp_lvl : compression level applied
*
* Returns
* -------
* number of bytes used in output buffer, negative error-code if failed.
*
*/
int64_t bshuf_compress_zstd(const void* in, void* out, const size_t size, const size_t
elem_size, size_t block_size, const int comp_lvl);
/* ---- bshuf_decompress_zstd ----
*
* Undo compression and bitshuffling.
*
* Decompress data then un-bitshuffle it in blocks of *block_size* elements.
*
* To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
* must patch the parameters used to compress the data.
*
* NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function
* ZSTD_decompress_fast from ZSTD, which does not protect against maliciously
* formed datasets. By modifying the compressed data, this function could be
* coerced into leaving the boundaries of the input buffer.
*
* Parameters
* ----------
* in : input buffer
* out : output buffer, must be of size * elem_size bytes
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Process in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* number of bytes consumed in *input* buffer, negative error-code if failed.
*
*/
int64_t bshuf_decompress_zstd(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size);
#endif // ZSTD_SUPPORT
#ifdef __cplusplus
} // extern "C"
#endif
#endif // BITSHUFFLE_H

2035
compression/bitshuffle/bitshuffle_core.c Normal file
View File

File diff suppressed because it is too large Load Diff

182
compression/bitshuffle/bitshuffle_core.h Normal file
View File

@@ -0,0 +1,182 @@
/*
* Bitshuffle - Filter for improving compression of typed binary data.
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*
* Header File
*
* Worker routines return an int64_t which is the number of bytes processed
* if positive or an error code if negative.
*
* Error codes:
* -1 : Failed to allocate memory.
* -11 : Missing SSE.
* -12 : Missing AVX.
* -13 : Missing Arm Neon.
* -14 : Missing AVX512.
* -80 : Input size not a multiple of 8.
* -81 : block_size not multiple of 8.
* -91 : Decompression error, wrong number of bytes processed.
* -1YYY : Error internal to compression routine with error code -YYY.
*/
#ifndef BITSHUFFLE_CORE_H
#define BITSHUFFLE_CORE_H
// We assume GNU g++ defining `__cplusplus` has stdint.h
#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
#include <stdint.h>
#else
typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
typedef signed int int32_t;
typedef unsigned long long uint64_t;
typedef long long int64_t;
#endif
#include <stdlib.h>
// These are usually set in the setup.py.
#ifndef BSHUF_VERSION_MAJOR
#define BSHUF_VERSION_MAJOR 0
#define BSHUF_VERSION_MINOR 4
#define BSHUF_VERSION_POINT 0
#endif
#ifdef __cplusplus
extern "C" {
#endif
/* --- bshuf_using_SSE2 ----
*
* Whether routines where compiled with the SSE2 instruction set.
*
* Returns
* -------
* 1 if using SSE2, 0 otherwise.
*
*/
int bshuf_using_SSE2(void);
/* ---- bshuf_using_NEON ----
*
* Whether routines where compiled with the NEON instruction set.
*
* Returns
* -------
* 1 if using NEON, 0 otherwise.
*
*/
int bshuf_using_NEON(void);
/* ---- bshuf_using_AVX2 ----
*
* Whether routines where compiled with the AVX2 instruction set.
*
* Returns
* -------
* 1 if using AVX2, 0 otherwise.
*
*/
int bshuf_using_AVX2(void);
/* ---- bshuf_using_AVX512 ----
*
* Whether routines where compiled with the AVX512 instruction set.
*
* Returns
* -------
* 1 if using AVX512, 0 otherwise.
*
*/
int bshuf_using_AVX512(void);
/* ---- bshuf_default_block_size ----
*
* The default block size as function of element size.
*
* This is the block size used by the blocked routines (any routine
* taking a *block_size* argument) when the block_size is not provided
* (zero is passed).
*
* The results of this routine are guaranteed to be stable such that
* shuffled/compressed data can always be decompressed.
*
* Parameters
* ----------
* elem_size : element size of data to be shuffled/compressed.
*
*/
size_t bshuf_default_block_size(const size_t elem_size);
/* ---- bshuf_bitshuffle ----
*
* Bitshuffle the data.
*
* Transpose the bits within elements, in blocks of *block_size*
* elements.
*
* Parameters
* ----------
* in : input buffer, must be of size * elem_size bytes
* out : output buffer, must be of size * elem_size bytes
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Do transpose in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* number of bytes processed, negative error-code if failed.
*
*/
int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size);
/* ---- bshuf_bitunshuffle ----
*
* Unshuffle bitshuffled data.
*
* Untranspose the bits within elements, in blocks of *block_size*
* elements.
*
* To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
* must match the parameters used to shuffle the data.
*
* Parameters
* ----------
* in : input buffer, must be of size * elem_size bytes
* out : output buffer, must be of size * elem_size bytes
* size : number of elements in input
* elem_size : element size of typed data
* block_size : Do transpose in blocks of this many elements. Pass 0 to
* select automatically (recommended).
*
* Returns
* -------
* number of bytes processed, negative error-code if failed.
*
*/
int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size,
const size_t elem_size, size_t block_size);
#ifdef __cplusplus
} // extern "C"
#endif
#endif // BITSHUFFLE_CORE_H

75
compression/bitshuffle/bitshuffle_internals.h Normal file
View File

@@ -0,0 +1,75 @@
/*
* Bitshuffle - Filter for improving compression of typed binary data.
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*/
#ifndef BITSHUFFLE_INTERNALS_H
#define BITSHUFFLE_INTERNALS_H
// We assume GNU g++ defining `__cplusplus` has stdint.h
#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
#include <stdint.h>
#else
typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
typedef signed int int32_t;
typedef unsigned long long uint64_t;
typedef long long int64_t;
#endif
#include <stdlib.h>
#include "iochain.h"
// Constants.
#ifndef BSHUF_MIN_RECOMMEND_BLOCK
#define BSHUF_MIN_RECOMMEND_BLOCK 128
#define BSHUF_BLOCKED_MULT 8 // Block sizes must be multiple of this.
#define BSHUF_TARGET_BLOCK_SIZE_B 8192
#endif
// Macros.
#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; }
#ifdef __cplusplus
extern "C" {
#endif
/* ---- Utility functions for internal use only ---- */
int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size);
/* Read a 32 bit unsigned integer from a buffer big endian order. */
uint32_t bshuf_read_uint32_BE(const void* buf);
/* Write a 32 bit unsigned integer to a buffer in big endian order. */
void bshuf_write_uint32_BE(void* buf, uint32_t num);
int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size);
/* Function definition for worker functions that process a single block. */
typedef int64_t (*bshufBlockFunDef)(ioc_chain* C_ptr,
const size_t size, const size_t elem_size, const int option);
/* Wrap a function for processing a single block to process an entire buffer in
* parallel. */
int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,
const size_t size, const size_t elem_size, size_t block_size, const int option);
#ifdef __cplusplus
} // extern "C"
#endif
#endif // BITSHUFFLE_INTERNALS_H

260
compression/bitshuffle/bshuf_h5filter.c Normal file
View File

@@ -0,0 +1,260 @@
/*
* Bitshuffle HDF5 filter
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*/
#include "bitshuffle.h"
#include "bshuf_h5filter.h"
#define PUSH_ERR(func, minor, str) \
H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str)
// Prototypes from bitshuffle.c
void bshuf_write_uint64_BE(void* buf, uint64_t num);
uint64_t bshuf_read_uint64_BE(void* buf);
void bshuf_write_uint32_BE(void* buf, uint32_t num);
uint32_t bshuf_read_uint32_BE(const void* buf);
// Only called on compresion, not on reverse.
herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){
herr_t r;
size_t ii;
unsigned int elem_size;
unsigned int flags;
size_t nelements = 8;
size_t nelem_max = 11;
unsigned values[] = {0,0,0,0,0,0,0,0,0,0,0};
unsigned tmp_values[] = {0,0,0,0,0,0,0,0};
char msg[80];
r = H5Pget_filter_by_id2(dcpl, BSHUF_H5FILTER, &flags, &nelements,
tmp_values, 0, NULL, NULL);
if(r<0) return -1;
// First 3 slots reserved. Move any passed options to higher addresses.
for (ii=0; ii < nelements && ii + 3 < nelem_max; ii++) {
values[ii + 3] = tmp_values[ii];
}
nelements = 3 + nelements;
values[0] = BSHUF_VERSION_MAJOR;
values[1] = BSHUF_VERSION_MINOR;
elem_size = H5Tget_size(type);
if(elem_size <= 0) {
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid element size.");
return -1;
}
values[2] = elem_size;
// Validate user supplied arguments.
if (nelements > 3) {
if (values[3] % 8 || values[3] < 0) {
sprintf(msg, "Error in bitshuffle. Invalid block size: %d.",
values[3]);
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK, msg);
return -1;
}
}
if (nelements > 4) {
switch (values[4]) {
case 0:
break;
case BSHUF_H5_COMPRESS_LZ4:
break;
#ifdef ZSTD_SUPPORT
case BSHUF_H5_COMPRESS_ZSTD:
break;
#endif
default:
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid bitshuffle compression.");
}
}
r = H5Pmodify_filter(dcpl, BSHUF_H5FILTER, flags, nelements, values);
if(r<0) return -1;
return 1;
}
size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf) {
size_t size, elem_size;
int err = -1;
char msg[80];
size_t block_size = 0;
size_t buf_size_out, nbytes_uncomp, nbytes_out;
char* in_buf = *buf;
void *out_buf;
if (cd_nelmts < 3) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Not enough parameters.");
return 0;
}
elem_size = cd_values[2];
#ifdef ZSTD_SUPPORT
const int comp_lvl = cd_values[5];
#endif
// User specified block size.
if (cd_nelmts > 3) block_size = cd_values[3];
if (block_size == 0) block_size = bshuf_default_block_size(elem_size);
#ifndef ZSTD_SUPPORT
if (cd_nelmts > 4 && (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD)) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"ZSTD compression filter chosen but ZSTD support not installed.");
return 0;
}
#endif
// Compression in addition to bitshuffle.
if (cd_nelmts > 4 && (cd_values[4] == BSHUF_H5_COMPRESS_LZ4 || cd_values[4] == BSHUF_H5_COMPRESS_ZSTD)) {
if (flags & H5Z_FLAG_REVERSE) {
// First eight bytes is the number of bytes in the output buffer,
// little endian.
nbytes_uncomp = bshuf_read_uint64_BE(in_buf);
// Override the block size with the one read from the header.
block_size = bshuf_read_uint32_BE((const char*) in_buf + 8) / elem_size;
// Skip over the header.
in_buf += 12;
buf_size_out = nbytes_uncomp;
} else {
nbytes_uncomp = nbytes;
// Pick which compressions library to use
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#endif
}
} else {
nbytes_uncomp = nbytes;
buf_size_out = nbytes;
}
// TODO, remove this restriction by memcopying the extra.
if (nbytes_uncomp % elem_size) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Non integer number of elements.");
return 0;
}
size = nbytes_uncomp / elem_size;
out_buf = malloc(buf_size_out);
if (out_buf == NULL) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Could not allocate output buffer.");
return 0;
}
if (cd_nelmts > 4 && (cd_values[4] == BSHUF_H5_COMPRESS_LZ4 || cd_values[4] == BSHUF_H5_COMPRESS_ZSTD)) {
if (flags & H5Z_FLAG_REVERSE) {
// Bit unshuffle/decompress.
// Pick which compressions library to use
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
err = bshuf_decompress_lz4(in_buf, out_buf, size, elem_size, block_size);
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
err = bshuf_decompress_zstd(in_buf, out_buf, size, elem_size, block_size);
}
#endif
nbytes_out = nbytes_uncomp;
} else {
// Bit shuffle/compress.
// Write the header, described in
// http://www.hdfgroup.org/services/filters/HDF5_LZ4.pdf.
// Techincally we should be using signed integers instead of
// unsigned ones, however for valid inputs (positive numbers) these
// have the same representation.
bshuf_write_uint64_BE(out_buf, nbytes_uncomp);
bshuf_write_uint32_BE((char*) out_buf + 8, block_size * elem_size);
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
err = bshuf_compress_lz4(in_buf, (char*) out_buf + 12, size,
elem_size, block_size);
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
err = bshuf_compress_zstd(in_buf, (char*) out_buf + 12, size,
elem_size, block_size, comp_lvl);
}
#endif
nbytes_out = err + 12;
}
} else {
if (flags & H5Z_FLAG_REVERSE) {
// Bit unshuffle.
err = bshuf_bitunshuffle(in_buf, out_buf, size, elem_size,
block_size); } else {
// Bit shuffle.
err = bshuf_bitshuffle(in_buf, out_buf, size, elem_size,
block_size); } nbytes_out = nbytes; }
//printf("nb_in %d, nb_uncomp %d, nb_out %d, buf_out %d, block %d\n",
//nbytes, nbytes_uncomp, nbytes_out, buf_size_out, block_size);
if (err < 0) {
sprintf(msg, "Error in bitshuffle with error code %d.", err);
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK, msg);
free(out_buf);
return 0;
} else {
free(*buf);
*buf = out_buf;
*buf_size = buf_size_out;
return nbytes_out;
}
}
H5Z_class_t bshuf_H5Filter[1] = {{
H5Z_CLASS_T_VERS,
(H5Z_filter_t)(BSHUF_H5FILTER),
1, 1,
"bitshuffle; see https://github.com/kiyo-masui/bitshuffle",
NULL,
(H5Z_set_local_func_t)(bshuf_h5_set_local),
(H5Z_func_t)(bshuf_h5_filter)
}};
int bshuf_register_h5filter(void){
int retval;
retval = H5Zregister(bshuf_H5Filter);
if(retval<0){
PUSH_ERR("bshuf_register_h5filter",
H5E_CANTREGISTER, "Can't register bitshuffle filter");
}
return retval;
}

67
compression/bitshuffle/bshuf_h5filter.h Normal file
View File

@@ -0,0 +1,67 @@
/*
* Bitshuffle HDF5 filter
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*
* Header File
*
* Filter Options
* --------------
* block_size (option slot 0) : interger (optional)
* What block size to use (in elements not bytes). Default is 0,
* for which bitshuffle will pick a block size with a target of 8kb.
* Compression (option slot 1) : 0 or BSHUF_H5_COMPRESS_LZ4
* Whether to apply LZ4 compression to the data after bitshuffling.
* This is much faster than applying compression as a second filter
* because it is done when the small block of data is already in the
* L1 cache.
*
* For LZ4 compression, the compressed format of the data is the same as
* for the normal LZ4 filter described in
* http://www.hdfgroup.org/services/filters/HDF5_LZ4.pdf.
*
*/
#ifndef BSHUF_H5FILTER_H
#define BSHUF_H5FILTER_H
#ifdef __cplusplus
extern "C" {
#endif
#define H5Z_class_t_vers 2
#include "hdf5.h"
#define BSHUF_H5FILTER 32008
#define BSHUF_H5_COMPRESS_LZ4 2
#define BSHUF_H5_COMPRESS_ZSTD 3
extern H5Z_class_t bshuf_H5Filter[1];
/* ---- bshuf_register_h5filter ----
*
* Register the bitshuffle HDF5 filter within the HDF5 library.
*
* Call this before using the bitshuffle HDF5 filter from C unless
* using dynamically loaded filters.
*
*/
int bshuf_register_h5filter(void);
#ifdef __cplusplus
} // extern "C"
#endif
#endif // BSHUF_H5FILTER_H

90
compression/bitshuffle/iochain.c Normal file
View File

@@ -0,0 +1,90 @@
/*
* IOchain - Distribute a chain of dependant IO events amoung threads.
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*/
#include <stdlib.h>
#include "iochain.h"
void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0) {
#ifdef _OPENMP
omp_init_lock(&C->next_lock);
for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
omp_init_lock(&(C->in_pl[ii].lock));
omp_init_lock(&(C->out_pl[ii].lock));
}
#endif
C->next = 0;
C->in_pl[0].ptr = in_ptr_0;
C->out_pl[0].ptr = out_ptr_0;
}
void ioc_destroy(ioc_chain *C) {
#ifdef _OPENMP
omp_destroy_lock(&C->next_lock);
for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
omp_destroy_lock(&(C->in_pl[ii].lock));
omp_destroy_lock(&(C->out_pl[ii].lock));
}
#endif
}
const void * ioc_get_in(ioc_chain *C, size_t *this_iter) {
#ifdef _OPENMP
omp_set_lock(&C->next_lock);
#pragma omp flush
#endif
*this_iter = C->next;
C->next ++;
#ifdef _OPENMP
omp_set_lock(&(C->in_pl[*this_iter % IOC_SIZE].lock));
omp_set_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
omp_set_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
omp_unset_lock(&C->next_lock);
#endif
return C->in_pl[*this_iter % IOC_SIZE].ptr;
}
void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr) {
C->in_pl[(*this_iter + 1) % IOC_SIZE].ptr = in_ptr;
#ifdef _OPENMP
omp_unset_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
#endif
}
void * ioc_get_out(ioc_chain *C, size_t *this_iter) {
#ifdef _OPENMP
omp_set_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
#pragma omp flush
#endif
void *out_ptr = C->out_pl[*this_iter % IOC_SIZE].ptr;
#ifdef _OPENMP
omp_unset_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
#endif
return out_ptr;
}
void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) {
C->out_pl[(*this_iter + 1) % IOC_SIZE].ptr = out_ptr;
#ifdef _OPENMP
omp_unset_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
// *in_pl[this_iter]* lock released at the end of the iteration to avoid being
// overtaken by previous threads and having *out_pl[this_iter]* corrupted.
// Especially worried about thread 0, iteration 0.
omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock));
#endif
}

94
compression/bitshuffle/iochain.h Normal file
View File

@@ -0,0 +1,94 @@
/*
* IOchain - Distribute a chain of dependant IO events amoung threads.
*
* This file is part of Bitshuffle
* Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
* Website: http://www.github.com/kiyo-masui/bitshuffle
* Created: 2014
*
* See LICENSE file for details about copyright and rights to use.
*
*
* Header File
*
* Similar in concept to a queue. Each task includes reading an input
* and writing output, but the location of the input/output (the pointers)
* depend on the previous item in the chain.
*
* This is designed for parallelizing blocked compression/decompression IO,
* where the destination of a compressed block depends on the compressed size
* of all previous blocks.
*
* Implemented with OpenMP locks.
*
*
* Usage
* -----
* - Call `ioc_init` in serial block.
* - Each thread should create a local variable *size_t this_iter* and
* pass its address to all function calls. Its value will be set
* inside the functions and is used to identify the thread.
* - Each thread must call each of the `ioc_get*` and `ioc_set*` methods
* exactly once per iteration, starting with `ioc_get_in` and ending
* with `ioc_set_next_out`.
* - The order (`ioc_get_in`, `ioc_set_next_in`, *work*, `ioc_get_out`,
* `ioc_set_next_out`, *work*) is most efficient.
* - Have each thread call `ioc_end_pop`.
* - `ioc_get_in` is blocked until the previous entry's
* `ioc_set_next_in` is called.
* - `ioc_get_out` is blocked until the previous entry's
* `ioc_set_next_out` is called.
* - There are no blocks on the very first iteration.
* - Call `ioc_destroy` in serial block.
* - Safe for num_threads >= IOC_SIZE (but less efficient).
*
*/
#ifndef IOCHAIN_H
#define IOCHAIN_H
#include <stdlib.h>
#ifdef _OPENMP
#include <omp.h>
#endif
#define IOC_SIZE 33
typedef struct ioc_ptr_and_lock {
#ifdef _OPENMP
omp_lock_t lock;
#endif
void *ptr;
} ptr_and_lock;
typedef struct ioc_const_ptr_and_lock {
#ifdef _OPENMP
omp_lock_t lock;
#endif
const void *ptr;
} const_ptr_and_lock;
typedef struct ioc_chain {
#ifdef _OPENMP
omp_lock_t next_lock;
#endif
size_t next;
const_ptr_and_lock in_pl[IOC_SIZE];
ptr_and_lock out_pl[IOC_SIZE];
} ioc_chain;
void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0);
void ioc_destroy(ioc_chain *C);
const void * ioc_get_in(ioc_chain *C, size_t *this_iter);
void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr);
void * ioc_get_out(ioc_chain *C, size_t *this_iter);
void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr);
#endif // IOCHAIN_H

24
compression/lz4/LICENSE Normal file
View File

@@ -0,0 +1,24 @@
LZ4 Library
Copyright (c) 2011-2016, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

2722
compression/lz4/lz4.c Normal file
View File

File diff suppressed because it is too large Load Diff

842
compression/lz4/lz4.h Normal file
View File

@@ -0,0 +1,842 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
* Copyright (C) 2011-2020, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef LZ4_H_2983827168210
#define LZ4_H_2983827168210
/* --- Dependency --- */
#include <stddef.h> /* size_t */
/**
Introduction
LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
The LZ4 compression library provides in-memory compression and decompression functions.
It gives full buffer control to user.
Compression can be done in:
- a single step (described as Simple Functions)
- a single step, reusing a context (described in Advanced Functions)
- unbounded multiple steps (described as Streaming compression)
lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
Decompressing such a compressed block requires additional metadata.
Exact metadata depends on exact decompression function.
For the typical case of LZ4_decompress_safe(),
metadata includes block's compressed size, and maximum bound of decompressed size.
Each application is free to encode and pass such metadata in whichever way it wants.
lz4.h only handle blocks, it can not generate Frames.
Blocks are different from Frames (doc/lz4_Frame_format.md).
Frames bundle both blocks and metadata in a specified manner.
Embedding metadata is required for compressed data to be self-contained and portable.
Frame format is delivered through a companion API, declared in lz4frame.h.
The `lz4` CLI can only manage frames.
*/
/*^***************************************************************
* Export parameters
*****************************************************************/
/*
* LZ4_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
* LZ4LIB_VISIBILITY :
* Control library symbols visibility.
*/
#ifndef LZ4LIB_VISIBILITY
# if defined(__GNUC__) && (__GNUC__ >= 4)
# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))
# else
# define LZ4LIB_VISIBILITY
# endif
#endif
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define LZ4LIB_API LZ4LIB_VISIBILITY
#endif
/*! LZ4_FREESTANDING :
* When this macro is set to 1, it enables "freestanding mode" that is
* suitable for typical freestanding environment which doesn't support
* standard C library.
*
* - LZ4_FREESTANDING is a compile-time switch.
* - It requires the following macros to be defined:
* LZ4_memcpy, LZ4_memmove, LZ4_memset.
* - It only enables LZ4/HC functions which don't use heap.
* All LZ4F_* functions are not supported.
* - See tests/freestanding.c to check its basic setup.
*/
#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1)
# define LZ4_HEAPMODE 0
# define LZ4HC_HEAPMODE 0
# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1
# if !defined(LZ4_memcpy)
# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'."
# endif
# if !defined(LZ4_memset)
# error "LZ4_FREESTANDING requires macro 'LZ4_memset'."
# endif
# if !defined(LZ4_memmove)
# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'."
# endif
#elif ! defined(LZ4_FREESTANDING)
# define LZ4_FREESTANDING 0
#endif
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
#define LZ4_QUOTE(str) #str
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */
LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */
LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */
/*-************************************
* Tuning parameter
**************************************/
#define LZ4_MEMORY_USAGE_MIN 10
#define LZ4_MEMORY_USAGE_DEFAULT 14
#define LZ4_MEMORY_USAGE_MAX 20
/*!
* LZ4_MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
* Increasing memory usage improves compression ratio, at the cost of speed.
* Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#ifndef LZ4_MEMORY_USAGE
# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
#endif
#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
# error "LZ4_MEMORY_USAGE is too small !"
#endif
#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX)
# error "LZ4_MEMORY_USAGE is too large !"
#endif
/*-************************************
* Simple Functions
**************************************/
/*! LZ4_compress_default() :
* Compresses 'srcSize' bytes from buffer 'src'
* into already allocated 'dst' buffer of size 'dstCapacity'.
* Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
* It also runs faster, so it's a recommended setting.
* If the function cannot compress 'src' into a more limited 'dst' budget,
* compression stops *immediately*, and the function result is zero.
* In which case, 'dst' content is undefined (invalid).
* srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
* dstCapacity : size of buffer 'dst' (which must be already allocated)
* @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
* or 0 if compression fails
* Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
*/
LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
/*! LZ4_decompress_safe() :
* compressedSize : is the exact complete size of the compressed block.
* dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size.
* @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
* Note 1 : This function is protected against malicious data packets :
* it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
* even if the compressed block is maliciously modified to order the decoder to do these actions.
* In such case, the decoder stops immediately, and considers the compressed block malformed.
* Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
* The implementation is free to send / store / derive this information in whichever way is most beneficial.
* If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
*/
LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
/*-************************************
* Advanced Functions
**************************************/
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
/*! LZ4_compressBound() :
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is incorrect (too large or negative)
*/
LZ4LIB_API int LZ4_compressBound(int inputSize);
/*! LZ4_compress_fast() :
Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c).
Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c).
*/
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_compress_fast_extState() :
* Same as LZ4_compress_fast(), using an externally allocated memory space for its state.
* Use LZ4_sizeofState() to know how much memory must be allocated,
* and allocate it on 8-bytes boundaries (using `malloc()` typically).
* Then, provide this buffer as `void* state` to compression function.
*/
LZ4LIB_API int LZ4_sizeofState(void);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_compress_destSize() :
* Reverse the logic : compresses as much data as possible from 'src' buffer
* into already allocated buffer 'dst', of size >= 'targetDestSize'.
* This function either compresses the entire 'src' content into 'dst' if it's large enough,
* or fill 'dst' buffer completely with as much data as possible from 'src'.
* note: acceleration parameter is fixed to "default".
*
* *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
* New value is necessarily <= input value.
* @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
* or 0 if compression fails.
*
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+):
* the produced compressed content could, in specific circumstances,
* require to be decompressed into a destination buffer larger
* by at least 1 byte than the content to decompress.
* If an application uses `LZ4_compress_destSize()`,
* it's highly recommended to update liblz4 to v1.9.2 or better.
* If this can't be done or ensured,
* the receiving decompression function should provide
* a dstCapacity which is > decompressedSize, by at least 1 byte.
* See https://github.com/lz4/lz4/issues/859 for details
*/
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
/*! LZ4_decompress_safe_partial() :
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
* into destination buffer 'dst' of size 'dstCapacity'.
* Up to 'targetOutputSize' bytes will be decoded.
* The function stops decoding on reaching this objective.
* This can be useful to boost performance
* whenever only the beginning of a block is required.
*
* @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize)
* If source stream is detected malformed, function returns a negative result.
*
* Note 1 : @return can be < targetOutputSize, if compressed block contains less data.
*
* Note 2 : targetOutputSize must be <= dstCapacity
*
* Note 3 : this function effectively stops decoding on reaching targetOutputSize,
* so dstCapacity is kind of redundant.
* This is because in older versions of this function,
* decoding operation would still write complete sequences.
* Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize,
* it could write more bytes, though only up to dstCapacity.
* Some "margin" used to be required for this operation to work properly.
* Thankfully, this is no longer necessary.
* The function nonetheless keeps the same signature, in an effort to preserve API compatibility.
*
* Note 4 : If srcSize is the exact size of the block,
* then targetOutputSize can be any value,
* including larger than the block's decompressed size.
* The function will, at most, generate block's decompressed size.
*
* Note 5 : If srcSize is _larger_ than block's compressed size,
* then targetOutputSize **MUST** be <= block's decompressed size.
* Otherwise, *silent corruption will occur*.
*/
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
/*-*********************************************
* Streaming Compression Functions
***********************************************/
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
/**
Note about RC_INVOKED
- RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio).
https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros
- Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars)
and reports warning "RC4011: identifier truncated".
- To eliminate the warning, we surround long preprocessor symbol with
"#if !defined(RC_INVOKED) ... #endif" block that means
"skip this block when rc.exe is trying to read it".
*/
#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
#endif
/*! LZ4_resetStream_fast() : v1.9.0+
* Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
* (e.g., LZ4_compress_fast_continue()).
*
* An LZ4_stream_t must be initialized once before usage.
* This is automatically done when created by LZ4_createStream().
* However, should the LZ4_stream_t be simply declared on stack (for example),
* it's necessary to initialize it first, using LZ4_initStream().
*
* After init, start any new stream with LZ4_resetStream_fast().
* A same LZ4_stream_t can be re-used multiple times consecutively
* and compress multiple streams,
* provided that it starts each new stream with LZ4_resetStream_fast().
*
* LZ4_resetStream_fast() is much faster than LZ4_initStream(),
* but is not compatible with memory regions containing garbage data.
*
* Note: it's only useful to call LZ4_resetStream_fast()
* in the context of streaming compression.
* The *extState* functions perform their own resets.
* Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive.
*/
LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
/*! LZ4_loadDict() :
* Use this function to reference a static dictionary into LZ4_stream_t.
* The dictionary must remain available during compression.
* LZ4_loadDict() triggers a reset, so any previous data will be forgotten.
* The same dictionary will have to be loaded on decompression side for successful decoding.
* Dictionary are useful for better compression of small data (KB range).
* While LZ4 accept any input as dictionary,
* results are generally better when using Zstandard's Dictionary Builder.
* Loading a size of 0 is allowed, and is the same as reset.
* @return : loaded dictionary size, in bytes (necessarily <= 64 KB)
*/
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! LZ4_compress_fast_continue() :
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
* 'dst' buffer must be already allocated.
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
*
* @return : size of compressed block
* or 0 if there is an error (typically, cannot fit into 'dst').
*
* Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
* Each block has precise boundaries.
* Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata.
* It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
*
* Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory !
*
* Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
* Make sure that buffers are separated, by at least one byte.
* This construction ensures that each block only depends on previous block.
*
* Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
*
* Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed.
*/
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_saveDict() :
* If last 64KB data cannot be guaranteed to remain available at its current memory location,
* save it into a safer place (char* safeBuffer).
* This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
* but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
* @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
*/
LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
/*-**********************************************
* Streaming Decompression Functions
* Bufferless synchronous API
************************************************/
typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */
/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
* creation / destruction of streaming decompression tracking context.
* A tracking context can be re-used multiple times.
*/
#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */
#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION)
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */
#endif
/*! LZ4_setStreamDecode() :
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
* Use this function to start decompression of a new stream of blocks.
* A dictionary can optionally be set. Use NULL or size 0 for a reset order.
* Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
* @return : 1 if OK, 0 if error
*/
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
/*! LZ4_decoderRingBufferSize() : v1.8.2+
* Note : in a ring buffer scenario (optional),
* blocks are presumed decompressed next to each other
* up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
* at which stage it resumes from beginning of ring buffer.
* When setting such a ring buffer for streaming decompression,
* provides the minimum size of this ring buffer
* to be compatible with any source respecting maxBlockSize condition.
* @return : minimum ring buffer size,
* or 0 if there is an error (invalid maxBlockSize).
*/
LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */
/*! LZ4_decompress_*_continue() :
* These decoding functions allow decompression of consecutive blocks in "streaming" mode.
* A block is an unsplittable entity, it must be presented entirely to a decompression function.
* Decompression functions only accepts one block at a time.
* The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
* If less than 64KB of data has been decoded, all the data must be present.
*
* Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
* - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
* maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
* In which case, encoding and decoding buffers do not need to be synchronized.
* Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
* - Synchronized mode :
* Decompression buffer size is _exactly_ the same as compression buffer size,
* and follows exactly same update rule (block boundaries at same positions),
* and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
* _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
* - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
* In which case, encoding and decoding buffers do not need to be synchronized,
* and encoding ring buffer can have any size, including small ones ( < 64 KB).
*
* Whenever these conditions are not possible,
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
*/
LZ4LIB_API int
LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode,
const char* src, char* dst,
int srcSize, int dstCapacity);
/*! LZ4_decompress_*_usingDict() :
* These decoding functions work the same as
* a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
* They are stand-alone, and don't need an LZ4_streamDecode_t structure.
* Dictionary is presumed stable : it must remain accessible and unmodified during decompression.
* Performance tip : Decompression speed can be substantially increased
* when dst == dictStart + dictSize.
*/
LZ4LIB_API int
LZ4_decompress_safe_usingDict(const char* src, char* dst,
int srcSize, int dstCapacity,
const char* dictStart, int dictSize);
LZ4LIB_API int
LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
int compressedSize,
int targetOutputSize, int maxOutputSize,
const char* dictStart, int dictSize);
#endif /* LZ4_H_2983827168210 */
/*^*************************************
* !!!!!! STATIC LINKING ONLY !!!!!!
***************************************/
/*-****************************************************************************
* Experimental section
*
* Symbols declared in this section must be considered unstable. Their
* signatures or semantics may change, or they may be removed altogether in the
* future. They are therefore only safe to depend on when the caller is
* statically linked against the library.
*
* To protect against unsafe usage, not only are the declarations guarded,
* the definitions are hidden by default
* when building LZ4 as a shared/dynamic library.
*
* In order to access these declarations,
* define LZ4_STATIC_LINKING_ONLY in your application
* before including LZ4's headers.
*
* In order to make their implementations accessible dynamically, you must
* define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
******************************************************************************/
#ifdef LZ4_STATIC_LINKING_ONLY
#ifndef LZ4_STATIC_3504398509
#define LZ4_STATIC_3504398509
#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
#define LZ4LIB_STATIC_API LZ4LIB_API
#else
#define LZ4LIB_STATIC_API
#endif
/*! LZ4_compress_fast_extState_fastReset() :
* A variant of LZ4_compress_fast_extState().
*
* Using this variant avoids an expensive initialization step.
* It is only safe to call if the state buffer is known to be correctly initialized already
* (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized").
* From a high level, the difference is that
* this function initializes the provided state with a call to something like LZ4_resetStream_fast()
* while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
*/
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_attach_dictionary() :
* This is an experimental API that allows
* efficient use of a static dictionary many times.
*
* Rather than re-loading the dictionary buffer into a working context before
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
* working LZ4_stream_t, this function introduces a no-copy setup mechanism,
* in which the working stream references the dictionary stream in-place.
*
* Several assumptions are made about the state of the dictionary stream.
* Currently, only streams which have been prepared by LZ4_loadDict() should
* be expected to work.
*
* Alternatively, the provided dictionaryStream may be NULL,
* in which case any existing dictionary stream is unset.
*
* If a dictionary is provided, it replaces any pre-existing stream history.
* The dictionary contents are the only history that can be referenced and
* logically immediately precede the data compressed in the first subsequent
* compression call.
*
* The dictionary will only remain attached to the working stream through the
* first compression call, at the end of which it is cleared. The dictionary
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the first compression call on the stream.
*/
LZ4LIB_STATIC_API void
LZ4_attach_dictionary(LZ4_stream_t* workingStream,
const LZ4_stream_t* dictionaryStream);
/*! In-place compression and decompression
*
* It's possible to have input and output sharing the same buffer,
* for highly constrained memory environments.
* In both cases, it requires input to lay at the end of the buffer,
* and decompression to start at beginning of the buffer.
* Buffer size must feature some margin, hence be larger than final size.
*
* |<------------------------buffer--------------------------------->|
* |<-----------compressed data--------->|
* |<-----------decompressed size------------------>|
* |<----margin---->|
*
* This technique is more useful for decompression,
* since decompressed size is typically larger,
* and margin is short.
*
* In-place decompression will work inside any buffer
* which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
* This presumes that decompressedSize > compressedSize.
* Otherwise, it means compression actually expanded data,
* and it would be more efficient to store such data with a flag indicating it's not compressed.
* This can happen when data is not compressible (already compressed, or encrypted).
*
* For in-place compression, margin is larger, as it must be able to cope with both
* history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
* and data expansion, which can happen when input is not compressible.
* As a consequence, buffer size requirements are much higher,
* and memory savings offered by in-place compression are more limited.
*
* There are ways to limit this cost for compression :
* - Reduce history size, by modifying LZ4_DISTANCE_MAX.
* Note that it is a compile-time constant, so all compressions will apply this limit.
* Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
* so it's a reasonable trick when inputs are known to be small.
* - Require the compressor to deliver a "maximum compressed size".
* This is the `dstCapacity` parameter in `LZ4_compress*()`.
* When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
* in which case, the return code will be 0 (zero).
* The caller must be ready for these cases to happen,
* and typically design a backup scheme to send data uncompressed.
* The combination of both techniques can significantly reduce
* the amount of margin required for in-place compression.
*
* In-place compression can work in any buffer
* which size is >= (maxCompressedSize)
* with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
* LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
* so it's possible to reduce memory requirements by playing with them.
*/
#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32)
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */
# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
#endif
#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
#endif /* LZ4_STATIC_3504398509 */
#endif /* LZ4_STATIC_LINKING_ONLY */
#ifndef LZ4_H_98237428734687
#define LZ4_H_98237428734687
/*-************************************************************
* Private Definitions
**************************************************************
* Do not use these definitions directly.
* They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
* Accessing members will expose user code to API and/or ABI break in future versions of the library.
**************************************************************/
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# include <stdint.h>
typedef int8_t LZ4_i8;
typedef uint8_t LZ4_byte;
typedef uint16_t LZ4_u16;
typedef uint32_t LZ4_u32;
#else
typedef signed char LZ4_i8;
typedef unsigned char LZ4_byte;
typedef unsigned short LZ4_u16;
typedef unsigned int LZ4_u32;
#endif
/*! LZ4_stream_t :
* Never ever use below internal definitions directly !
* These definitions are not API/ABI safe, and may change in future versions.
* If you need static allocation, declare or allocate an LZ4_stream_t object.
**/
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
LZ4_u32 hashTable[LZ4_HASH_SIZE_U32];
const LZ4_byte* dictionary;
const LZ4_stream_t_internal* dictCtx;
LZ4_u32 currentOffset;
LZ4_u32 tableType;
LZ4_u32 dictSize;
/* Implicit padding to ensure structure is aligned */
};
#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */
union LZ4_stream_u {
char minStateSize[LZ4_STREAM_MINSIZE];
LZ4_stream_t_internal internal_donotuse;
}; /* previously typedef'd to LZ4_stream_t */
/*! LZ4_initStream() : v1.9.0+
* An LZ4_stream_t structure must be initialized at least once.
* This is automatically done when invoking LZ4_createStream(),
* but it's not when the structure is simply declared on stack (for example).
*
* Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t.
* It can also initialize any arbitrary buffer of sufficient size,
* and will @return a pointer of proper type upon initialization.
*
* Note : initialization fails if size and alignment conditions are not respected.
* In which case, the function will @return NULL.
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
* Note3: Before v1.9.0, use LZ4_resetStream() instead
**/
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
/*! LZ4_streamDecode_t :
* Never ever use below internal definitions directly !
* These definitions are not API/ABI safe, and may change in future versions.
* If you need static allocation, declare or allocate an LZ4_streamDecode_t object.
**/
typedef struct {
const LZ4_byte* externalDict;
const LZ4_byte* prefixEnd;
size_t extDictSize;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
#define LZ4_STREAMDECODE_MINSIZE 32
union LZ4_streamDecode_u {
char minStateSize[LZ4_STREAMDECODE_MINSIZE];
LZ4_streamDecode_t_internal internal_donotuse;
} ; /* previously typedef'd to LZ4_streamDecode_t */
/*-************************************
* Obsolete Functions
**************************************/
/*! Deprecation warnings
*
* Deprecated functions make the compiler generate a warning when invoked.
* This is meant to invite users to update their source code.
* Should deprecation warnings be a problem, it is generally possible to disable them,
* typically with -Wno-deprecated-declarations for gcc
* or _CRT_SECURE_NO_WARNINGS in Visual.
*
* Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS
* before including the header file.
*/
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45))
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# else
# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler")
# define LZ4_DEPRECATED(message) /* disabled */
# endif
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
/*! Obsolete compression functions (since v1.7.3) */
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/*! Obsolete decompression functions (since v1.8.0) */
LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/* Obsolete streaming functions (since v1.7.0)
* degraded functionality; do not use!
*
* In order to perform streaming compression, these functions depended on data
* that is no longer tracked in the state. They have been preserved as well as
* possible: using them will still produce a correct output. However, they don't
* actually retain any history between compression calls. The compression ratio
* achieved will therefore be no better than compressing each chunk
* independently.
*/
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void);
LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
/*! Obsolete streaming decoding functions (since v1.7.0) */
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) :
* These functions used to be faster than LZ4_decompress_safe(),
* but this is no longer the case. They are now slower.
* This is because LZ4_decompress_fast() doesn't know the input size,
* and therefore must progress more cautiously into the input buffer to not read beyond the end of block.
* On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
* As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
*
* The last remaining LZ4_decompress_fast() specificity is that
* it can decompress a block without knowing its compressed size.
* Such functionality can be achieved in a more secure manner
* by employing LZ4_decompress_safe_partial().
*
* Parameters:
* originalSize : is the uncompressed size to regenerate.
* `dst` must be already allocated, its size must be >= 'originalSize' bytes.
* @return : number of bytes read from source buffer (== compressed size).
* The function expects to finish at block's end exactly.
* If the source stream is detected malformed, the function stops decoding and returns a negative result.
* note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
* However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
* Also, since match offsets are not validated, match reads from 'src' may underflow too.
* These issues never happen if input (compressed) data is correct.
* But they may happen if input data is invalid (error or intentional tampering).
* As a consequence, use these functions in trusted environments with trusted data **only**.
*/
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
/*! LZ4_resetStream() :
* An LZ4_stream_t structure must be initialized at least once.
* This is done with LZ4_initStream(), or LZ4_resetStream().
* Consider switching to LZ4_initStream(),
* invoking LZ4_resetStream() will trigger deprecation warnings in the future.
*/
LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
#endif /* LZ4_H_98237428734687 */
#if defined (__cplusplus)
}
#endif

1
compression/zstd Submodule

Submodule compression/zstd added at 19105bf710

8
detector_control/CMakeLists.txt Normal file
View File

@@ -0,0 +1,8 @@
ADD_SUBDIRECTORY(slsDetectorPackage)
ADD_EXECUTABLE(jfjoch_detector jfjoch_detector.cpp JFJochDetector.cpp JFJochDetector.h DetectorWrapper.cpp DetectorWrapper.h)
TARGET_LINK_LIBRARIES(jfjoch_detector CommonFunctions slsSupportShared slsDetectorShared)
INSTALL(TARGETS sls_detector_put sls_detector_get jfjoch_detector RUNTIME)
SET_PROPERTY(TARGET jfjoch_detector PROPERTY INTERPROCEDURAL_OPTIMIZATION True)

201
detector_control/DetectorWrapper.cpp Normal file
View File

@@ -0,0 +1,201 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <thread>
#include "DetectorWrapper.h"
#include "../common/JFJochException.h"
#include "../common/Definitions.h"
void DetectorWrapper::Configure(const JFJochProtoBuf::DetectorConfig &request) {
logger.Info("Configure");
if (det.size() != request.modules_size()) {
logger.Error("Discrepancy in module number between DAQ and detector");
throw JFJochException(JFJochExceptionCategory::Detector,
"Discrepancy in module number between DAQ and detector");
}
try {
InternalStop();
det.setNumberofUDPInterfaces(2);
for (int i = 0; i < request.modules_size(); i++) {
auto &cfg = request.modules(i);
det.setSourceUDPIP(sls::IpAddr(cfg.ipv4_src_addr_1()), {i});
det.setSourceUDPIP2(sls::IpAddr(cfg.ipv4_src_addr_2()), {i});
det.setSourceUDPMAC(sls::MacAddr(BASE_DETECTOR_MAC + i * 2), {i});
det.setSourceUDPMAC2(sls::MacAddr(BASE_DETECTOR_MAC + i * 2 + 1), {i});
det.setDestinationUDPPort (cfg.udp_dest_port_1(), i);
det.setDestinationUDPPort2(cfg.udp_dest_port_2(), i);
det.setDestinationUDPIP( sls::IpAddr(cfg.ipv4_dest_addr_1()), {i});
det.setDestinationUDPIP2( sls::IpAddr(cfg.ipv4_dest_addr_2()), {i});
det.setDestinationUDPMAC( sls::MacAddr(cfg.mac_addr_dest_1()), {i});
det.setDestinationUDPMAC2(sls::MacAddr(cfg.mac_addr_dest_2()), {i});
}
det.setTemperatureControl(true);
det.setThresholdTemperature(55);
det.clearBit(0x4f, 15, {0});
det.clearBit(0x4e, 4);
det.setTimingMode(slsDetectorDefs::timingMode::TRIGGER_EXPOSURE);
det.setAutoComparatorDisable(true);
if (!det.getPowerChip().squash(false)) {
det.setPowerChip(true);
std::this_thread::sleep_for(std::chrono::seconds(5));
}
det.setHighVoltage(HIGH_VOLTAGE);
} catch (const std::exception &e) {
logger.ErrorException(e);
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
logger.Info(" ... done");
}
void DetectorWrapper::Start(const JFJochProtoBuf::DetectorInput &request) {
logger.Info("Start");
if (det.size() != request.modules_num())
throw JFJochException(JFJochExceptionCategory::Detector,
"Discrepancy in module number between DAQ and detector");
try {
InternalStop();
switch (request.mode()) {
case JFJochProtoBuf::PEDESTAL_G1:
det.setGainMode(slsDetectorDefs::gainMode::FORCE_SWITCH_G1);
break;
case JFJochProtoBuf::PEDESTAL_G2:
det.setGainMode(slsDetectorDefs::gainMode::FORCE_SWITCH_G2);
break;
default:
det.setGainMode(slsDetectorDefs::gainMode::DYNAMIC);
break;
}
det.setNextFrameNumber(1);
det.setNumberOfFrames(request.num_frames());
det.setNumberOfTriggers(request.num_triggers());
det.setStorageCellStart(request.storage_cell_start());
det.setNumberOfAdditionalStorageCells(request.storage_cell_number() - 1);
det.setStorageCellDelay(std::chrono::microseconds(request.storage_cell_delay()));
if (request.period_us() < MIN_FRAME_TIME_HALF_SPEED_IN_US)
det.setReadoutSpeed(slsDetectorDefs::speedLevel::FULL_SPEED);
else
det.setReadoutSpeed(slsDetectorDefs::speedLevel::HALF_SPEED);
det.setPeriod(std::chrono::microseconds(request.period_us()));
det.setExptime(std::chrono::microseconds(request.count_time_us()));
// Always synchronize to master
det.setBit(0x4f, 15, {0});
det.setBit(0x4e, 4);
det.startDetector();
} catch (std::exception &e) {
logger.ErrorException(e);
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
logger.Info(" ... done");
}
void DetectorWrapper::InternalStop() {
// Assume it is executed in try-catch!
auto state = GetState();
if (state == DetectorState::ERROR)
throw JFJochException(JFJochExceptionCategory::Detector,
"Detector in error state");
else if (state == DetectorState::BUSY) {
det.stopDetector();
std::this_thread::sleep_for(std::chrono::milliseconds(10));
state = GetState();
if (state != DetectorState::IDLE)
throw JFJochException(JFJochExceptionCategory::Detector,
"Detector busy and cannot be stopped");
}
}
void DetectorWrapper::Deactivate() {
logger.Info("Deactivate");
try {
InternalStop();
det.setHighVoltage(0);
std::this_thread::sleep_for(std::chrono::seconds(5));
det.setPowerChip(false);
} catch (std::exception &e) {
logger.ErrorException(e);
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
logger.Info(" ... done");
}
void DetectorWrapper::Stop() {
logger.Info("Stop");
try {
InternalStop();
} catch (std::exception &e) {
logger.ErrorException(e);
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
logger.Info(" ... done");
}
void DetectorWrapper::Trigger() {
logger.Info("Trigger");
try {
for (int i = 1; i < det.size(); i++)
det.setBit(0x4f, 2, {i});
std::this_thread::sleep_for(std::chrono::milliseconds(100));
det.setBit(0x4f, 2, {0});
std::this_thread::sleep_for(std::chrono::milliseconds(100));
det.clearBit(0x4f, 2);
} catch (std::exception &e) {
logger.ErrorException(e);
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
logger.Info(" ... done");
}
DetectorWrapper::DetectorState DetectorWrapper::GetState() const {
try {
bool is_idle = true;
for (auto & i : det.getDetectorStatus()) {
if (i == slsDetectorDefs::runStatus::ERROR)
return DetectorState::ERROR;
if ((i != slsDetectorDefs::runStatus::IDLE) &&
(i != slsDetectorDefs::runStatus::STOPPED) &&
(i != slsDetectorDefs::runStatus::RUN_FINISHED))
is_idle = false;
}
if (is_idle)
return DetectorState::IDLE;
else
return DetectorState::BUSY;
} catch (std::exception &e) {
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
}
int64_t DetectorWrapper::GetFirmwareVersion() {
try {
auto result = det.getFirmwareVersion();
return result.squash(0x0);
} catch (std::exception &e) {
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
}
int64_t DetectorWrapper::GetDetectorServerVersion() {
try {
auto result = det.getDetectorServerVersion();
return result.squash(0x0);
} catch (std::exception &e) {
throw JFJochException(JFJochExceptionCategory::Detector, e.what());
}
}

31
detector_control/DetectorWrapper.h Normal file
View File

@@ -0,0 +1,31 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef JUNGFRAUJOCH_DETECTORWRAPPER_H
#define JUNGFRAUJOCH_DETECTORWRAPPER_H
#include <sls/Detector.h>
#include <jfjoch.pb.h>
#include "../common/Logger.h"
#define BASE_DETECTOR_MAC 0xAABBCCDDEE10 // little-endian!
#define HIGH_VOLTAGE 120
class DetectorWrapper {
Logger logger{"DetectorWrapper"};
sls::Detector det;
void InternalStop();
public:
enum class DetectorState {IDLE, ERROR, BUSY};
[[nodiscard]] DetectorState GetState() const;
void Configure(const JFJochProtoBuf::DetectorConfig &config);
void Start(const JFJochProtoBuf::DetectorInput &request);
void Stop();
void Trigger();
void Deactivate();
int64_t GetFirmwareVersion();
int64_t GetDetectorServerVersion();
};
#endif //JUNGFRAUJOCH_DETECTORWRAPPER_H

86
detector_control/JFJochDetector.cpp Normal file
View File

@@ -0,0 +1,86 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include "JFJochDetector.h"
#include "../common/JFJochException.h"
grpc::Status JFJochDetector::Start(grpc::ServerContext *context, const JFJochProtoBuf::DetectorInput *request,
JFJochProtoBuf::Empty *response) {
std::unique_lock<std::mutex> ul(m);
try {
detector.Start(*request);
return grpc::Status::OK;
} catch (JFJochException &e) {
return {grpc::StatusCode::ABORTED, e.what()};
}
}
grpc::Status JFJochDetector::Stop(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
std::unique_lock<std::mutex> ul(m);
try {
detector.Stop();
return grpc::Status::OK;
} catch (JFJochException &e) {
return {grpc::StatusCode::ABORTED, e.what()};
}
}
grpc::Status JFJochDetector::On(grpc::ServerContext *context, const JFJochProtoBuf::DetectorConfig *request,
JFJochProtoBuf::Empty *response) {
std::unique_lock<std::mutex> ul(m);
try {
detector.Configure(*request);
return grpc::Status::OK;
} catch (JFJochException &e) {
return {grpc::StatusCode::ABORTED, e.what()};
}
}
grpc::Status JFJochDetector::Off(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
std::unique_lock<std::mutex> ul(m);
try {
detector.Deactivate();
return grpc::Status::OK;
} catch (JFJochException &e) {
return {grpc::StatusCode::ABORTED, e.what()};
}
}
grpc::Status JFJochDetector::Status(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::DetectorStatus *response) {
try {
switch(detector.GetState()) {
case DetectorWrapper::DetectorState::IDLE:
response->set_state(JFJochProtoBuf::IDLE);
break;
case DetectorWrapper::DetectorState::ERROR:
response->set_state(JFJochProtoBuf::ERROR);
break;
case DetectorWrapper::DetectorState::BUSY:
response->set_state(JFJochProtoBuf::BUSY);
break;
}
response->set_fw_version(detector.GetFirmwareVersion());
response->set_server_version(detector.GetDetectorServerVersion());
return grpc::Status::OK;
} catch (JFJochException &e) {
return {grpc::StatusCode::ABORTED, e.what()};
}
}
grpc::Status JFJochDetector::Trigger(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) {
std::unique_lock<std::mutex> ul(m);
try {
detector.Trigger();
return grpc::Status::OK;
} catch (JFJochException &e) {
return {grpc::StatusCode::ABORTED, e.what()};
}
}

35
detector_control/JFJochDetector.h Normal file
View File

@@ -0,0 +1,35 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef DETECTORWRAPPER_H
#define DETECTORWRAPPER_H
#include <sls/Detector.h>
#include "jfjoch.grpc.pb.h"
#include "DetectorWrapper.h"
#include "../common/Logger.h"
#define BASE_DETECTOR_MAC 0xAABBCCDDEE10 // little-endian!
#define HIGH_VOLTAGE 120
class JFJochDetector final : public JFJochProtoBuf::gRPC_JFJochDetector::Service {
std::mutex m;
DetectorWrapper detector;
public:
grpc::Status Start(grpc::ServerContext *context, const JFJochProtoBuf::DetectorInput *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Stop(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Status(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::DetectorStatus *response) override;
grpc::Status On(grpc::ServerContext *context, const JFJochProtoBuf::DetectorConfig *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Off(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
grpc::Status Trigger(grpc::ServerContext *context, const JFJochProtoBuf::Empty *request,
JFJochProtoBuf::Empty *response) override;
};
#endif //JUNGFRAUJOCH_DETECTORWRAPPER_H

27
detector_control/jfjoch_detector.cpp Normal file
View File

@@ -0,0 +1,27 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <fstream>
#include <grpcpp/server.h>
#include <grpcpp/server_builder.h>
#include <grpcpp/security/server_credentials.h>
#include <nlohmann/json.hpp>
#include "JFJochDetector.h"
#include "../common/Logger.h"
#include "../grpc/gRPCServer_Template.h"
int main(int argc, char **argv) {
Logger logger("jfjoch_detector");
nlohmann::json input;
std::string grpc_addr = "unix:/opt/jfjoch/.jfjoch-detector";
if (argc == 2) grpc_addr = "0.0.0.0:" + std::string(argv[1]);
JFJochDetector service;
auto server = gRPCServer(grpc_addr, service);
logger.Info("gRPC configuration listening on " + grpc_addr);
server->Wait();
}

1
detector_control/slsDetectorPackage Submodule

Submodule detector_control/slsDetectorPackage added at da8e0060d3

1
etc/CMakeLists.txt Normal file
View File

@@ -0,0 +1 @@
INSTALL(FILES broker.json recv.json index.json writer.json DESTINATION etc)

42
etc/broker.json Normal file
View File

@@ -0,0 +1,42 @@
{
"ipv4_subnet": 85,
"writer": [
{
"addr_grpc": "10.10.1.204:5234",
"addr_zmq": "tcp://10.10.1.242:5402"
},
{
"addr_grpc": "10.10.1.204:5235",
"addr_zmq": "tcp://10.10.1.242:5403"
},
{
"addr_grpc": "10.10.1.204:5236",
"addr_zmq": "tcp://10.10.1.242:5404"
},
{
"addr_grpc": "10.10.1.204:5237",
"addr_zmq": "tcp://10.10.1.242:5405"
}
],
"receiver_addr": "unix:/opt/jfjoch/.jfjoch-fpga-receiver",
"detector_addr": "unix:/opt/jfjoch/.jfjoch-detector",
"frontend_directory": "/opt/jfjoch/frontend",
"indexer": {
"addr_grpc": "mx-jungfrau-1:5405",
"addr_zmq": "tcp://10.10.1.242:5401"
},
"source_name": "Swiss Light Source",
"source_name_short": "SLS",
"instrument_name": "VESPA",
"instrument_name_short": "VESPA",
"gain_file": [
"/opt/jfjoch/gainmaps/gainMaps_M352_2020-01-31.bin",
"/opt/jfjoch/gainmaps/gainMaps_M307_2020-01-20.bin",
"/opt/jfjoch/gainmaps/gainMaps_M264_2019-07-29.bin",
"/opt/jfjoch/gainmaps/gainMaps_M253_2019-07-29.bin",
"/opt/jfjoch/gainmaps/gainMaps_M351_2020-01-20.bin",
"/opt/jfjoch/gainmaps/gainMaps_M312_2020-01-20.bin",
"/opt/jfjoch/gainmaps/gainMaps_M261_2019-07-29.bin",
"/opt/jfjoch/gainmaps/gainMaps_M310_2021-01-26.bin"
]
}

3
etc/index.json Normal file
View File

@@ -0,0 +1,3 @@
{
"nthreads": 64
}

30
etc/recv.json Normal file
View File

@@ -0,0 +1,30 @@
{
"compression_threads" : 64,
"device": [
{
"id" : 1,
"type": "pcie"
},
{
"id" : 3,
"type": "pcie"
},
{
"id" : 0,
"type": "pcie"
},
{
"id" : 2,
"type": "pcie"
}
],
"tcp_send_buffer_size": 256000000,
"zmq_send_high_watermark": 500,
"image_zmq_addr": ["tcp://10.10.1.242:5402",
"tcp://10.10.1.242:5403",
"tcp://10.10.1.242:5404",
"tcp://10.10.1.242:5405"],
"preview_zmq_addr": "tcp://0.0.0.0:5400",
"spot_zmq_addr": "tcp://10.10.1.242:5401",
"grpc_addr": "unix:/opt/jfjoch/.jfjoch-fpga-receiver"
}

5
etc/writer.json Normal file
View File

@@ -0,0 +1,5 @@
{
"base_directory": "/home/leonarski_f/data",
"gid": -1,
"grpc_addr": [4567,4568,4569,4570]
}

13
frame_serialize/CMakeLists.txt Normal file
View File

@@ -0,0 +1,13 @@
ADD_LIBRARY(FrameSerialize STATIC
JFJochFrameSerializer.cpp JFJochFrameSerializer.h
JFJochFrameDeserializer.cpp JFJochFrameDeserializer.h
ImageMessage.h
tinycbor/src/cborencoder.c
tinycbor/src/cborencoder_close_container_checked.c
tinycbor/src/cborencoder_float.c
tinycbor/src/cborparser.c
tinycbor/src/cborparser_float.c
tinycbor/src/cborpretty.c
tinycbor/src/cborerrorstrings.c
tinycbor/src/cbor.h
tinycbor/src/tinycbor-version.h CborErr.h StartMessage.h EndMessage.h CborUtil.h)

Some files were not shown because too many files have changed in this diff Show More