diff --git a/common/DetectorSettings.cpp b/common/DetectorSettings.cpp index a4bf1c26e..2f30ccf63 100644 --- a/common/DetectorSettings.cpp +++ b/common/DetectorSettings.cpp @@ -13,9 +13,9 @@ DetectorSettings &DetectorSettings::InternalGeneratorEnable(bool input) { return *this; } -DetectorSettings &DetectorSettings::InternalGeneratorImages(int64_t input) { +DetectorSettings &DetectorSettings::InternalGeneratorImages(int64_t input, int64_t max_images) { check_min("Internal packet generator images", input, 1); - check_max("Internal packet generator images", input, 64); + check_max("Internal packet generator images", input, max_images); internal_fpga_packet_generator_images = input; return *this; } diff --git a/common/DetectorSettings.h b/common/DetectorSettings.h index dbd56f185..9a5034d65 100644 --- a/common/DetectorSettings.h +++ b/common/DetectorSettings.h @@ -37,7 +37,9 @@ class DetectorSettings { DetectorTiming timing = DetectorTiming::Trigger; public: DetectorSettings& InternalGeneratorEnable(bool input); - DetectorSettings& InternalGeneratorImages(int64_t input); + // The broker's limit; the firmware's own is FPGA_INTERNAL_GENERATOR_MAX_IMAGES + static constexpr int64_t DefaultMaxInternalGeneratorImages = 64; + DetectorSettings& InternalGeneratorImages(int64_t input, int64_t max_images = DefaultMaxInternalGeneratorImages); DetectorSettings& StorageCells(int64_t input); DetectorSettings& StorageCellStart(int64_t input); DetectorSettings& StorageCellDelay(const std::chrono::nanoseconds &input); diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index d31d4926e..e7917d146 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -1094,8 +1094,8 @@ float DiffractionExperiment::GetPhotonEnergyForConversion_keV() const { return GetIncidentEnergy_keV(); } -DiffractionExperiment &DiffractionExperiment::InternalPacketGeneratorImages(int64_t input) { - detector_settings.InternalGeneratorImages(input); +DiffractionExperiment &DiffractionExperiment::InternalPacketGeneratorImages(int64_t input, int64_t max_images) { + detector_settings.InternalGeneratorImages(input, max_images); return *this; } diff --git a/common/DiffractionExperiment.h b/common/DiffractionExperiment.h index e8e689642..f45e08fcd 100644 --- a/common/DiffractionExperiment.h +++ b/common/DiffractionExperiment.h @@ -119,7 +119,8 @@ public: DiffractionExperiment& ZMQPreviewPeriod(const std::optional &input); DiffractionExperiment& UseInternalPacketGenerator(bool input); - DiffractionExperiment& InternalPacketGeneratorImages(int64_t input); + DiffractionExperiment& InternalPacketGeneratorImages(int64_t input, + int64_t max_images = DetectorSettings::DefaultMaxInternalGeneratorImages); DiffractionExperiment& MaskModuleEdges(bool input); DiffractionExperiment& MaskChipEdges(bool input); diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 0edc53ff3..75688e87d 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -77,8 +77,10 @@ Measures rugnux live stills on the FPGA card(s), with a stored dataset standing jfjoch_fpga_stills_test [-i ] [-s ] [-N ] [-B ] [-C a,b,c,al,be,ga] [-X ] [-p ] [-F] [-K[]] ``` -- The dataset must be a stored JUNGFRAU dataset, assembled with 8/36 pixel module gaps (as Jungfraujoch - writes it); the module layout is derived from the image size. Each image is folded back into module +- The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement + and orientation): the layout is the file's - its NXmx module groups, else the modules the gaps of its pixel + mask mark (where the file does not give a module's readout direction, the module is read out from the + top-left corner of its box; the multipixels are symmetric, so the frames are the same). Each image is folded back into module frames, the inverse of the host's geometry transformation: the inter-chip pixels the assembled image spreads one raw pixel over (2, or 4 at a chip corner) are summed back into it (the split conserves the count); where one of them is an error, gap or masked pixel the raw pixel is an error pixel, where @@ -89,8 +91,11 @@ jfjoch_fpga_stills_test [-i ] [-s ] [-N ] [-B ] [-C `JFJochReceiverFPGA`'s own live stills, for comparison). Beam centre, distance, energy and the cell come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is written. -- The generator holds only as many images as its card memory (and the 64-image limit of the - generator setting) allows: the first ones of the file, cycled for the whole run of `-i` images +- Each card's generator holds module frames of its own modules only: as many images as its memory holds + module frames (less one, which the host keeps free) divided by the modules of that card - e.g. 128 module + frames: a 9M (18 modules) on one card 7 images, on two cards of 9 modules 14. The run takes the smallest + over the cards (and at most 128, the firmware's image counter; the broker's limit of 64 images does not + apply here) and prints the mapping. These are the first ones of the file, cycled for the whole run of `-i` images (default: as many as the file has). **Images repeat**, so throughput, latency, spots per image and indexing rate are meaningful, the merge and its statistics are not. The generator sends as fast as the card can, faster than a detector: images the host cannot keep up with are dropped by the puller @@ -103,7 +108,7 @@ jfjoch_fpga_stills_test [-i ] [-s ] [-N ] [-B ] [-C (`StillsGPUAnalysis::PREPASS_IMAGES`) in half its slots. - `-K` runs on the mock of the card (`MockAcquisitionDevice`, which returns the generator's frames as loaded) instead of `/dev/jfjoch*`, so the program runs without a card; the optional value is the mock's - generator memory in MiB. + generator memory in MiB (default 64 MiB = 128 module frames). - To compare with the offline analysis of the same images, run it with `-i` equal to the number of distinct images it reports (no repeats) and `rugnux --force-still -e ` or `jfjoch_stills_gpu_replay -n ` on the same file. diff --git a/fpga/pcie_driver/jfjoch_fpga.h b/fpga/pcie_driver/jfjoch_fpga.h index 73efecf64..ca619841f 100644 --- a/fpga/pcie_driver/jfjoch_fpga.h +++ b/fpga/pcie_driver/jfjoch_fpga.h @@ -58,6 +58,9 @@ typedef __u64 uint64_t; // MAX_MODULES_FPGA is read by FPGA setup scripts and must be decimal, no expression! #define MAX_MODULES_FPGA 32 #define MAX_FPGA_SUMMATION 256 +// frame_generator.cpp: images in its memory (frame_cell, 7 bits) and module frames it addresses (mem_cell, 10 bits) +#define FPGA_INTERNAL_GENERATOR_MAX_IMAGES 128 +#define FPGA_INTERNAL_GENERATOR_MAX_MODULE_FRAMES 1024 #define ADU_HISTO_BIN_WIDTH 32 #define ADU_HISTO_BIN_COUNT (65536/ ADU_HISTO_BIN_WIDTH) diff --git a/tests/RawToConvertedGeometryTest.cpp b/tests/RawToConvertedGeometryTest.cpp index 5fa679486..28efb2bad 100644 --- a/tests/RawToConvertedGeometryTest.cpp +++ b/tests/RawToConvertedGeometryTest.cpp @@ -518,37 +518,65 @@ TEST_CASE("RawToConvertedCoordinate_Coord","[RawToConvertedGeometry]") { REQUIRE(Catch::Approx(c.y) == CONVERTED_MODULE_LINES * 2 + 36 - 1 - 300.25 - 2); // line 300 becomes 302 due to 1 chip boundary } -TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry","[RawToConvertedGeometry]") { +namespace { // Assembled with the multipixels split, then squeezed back: the raw frames, where the split divides evenly - DiffractionExperiment x(DetJF(4, 2, 8, 36, true)); - x.PixelSigned(true).BitDepthImage(16); + void CheckSqueezeRoundTrip(const DetectorSetup &detector) { + DiffractionExperiment x(detector); + x.PixelSigned(true).BitDepthImage(16); - std::vector raw(x.GetModulesNum() * RAW_MODULE_SIZE); - for (size_t i = 0; i < raw.size(); i++) - raw[i] = static_cast(4 * (i % 5000) - 400); - raw[255 * RAW_MODULE_COLS + 255] = INT16_MIN; // error, chip corner - raw[RAW_MODULE_SIZE + 10 * RAW_MODULE_COLS + 256] = INT16_MAX; // saturated, multipixel + std::vector raw(x.GetModulesNum() * RAW_MODULE_SIZE); + for (size_t i = 0; i < raw.size(); i++) + raw[i] = static_cast(4 * (i % 5000) - 400); + raw[255 * RAW_MODULE_COLS + 255] = INT16_MIN; // error, chip corner + raw[RAW_MODULE_SIZE + 10 * RAW_MODULE_COLS + 256] = INT16_MAX; // saturated, multipixel - std::vector image(x.GetPixelsNum(), INT32_MIN); - for (int m = 0; m < x.GetModulesNum(); m++) - TransferModuleAdjustMultipixels(image.data(), raw.data() + m * RAW_MODULE_SIZE, - x.GetModuleSlowDirectionStep(m), INT16_MIN, INT16_MAX, - x.GetModuleFastDirectionStep(m), x.GetPixel0OfModuleConv(m)); - for (auto &v: image) { - if (v == INT16_MIN) - v = INT32_MIN; - else if (v == INT16_MAX) - v = INT32_MAX; + std::vector image(x.GetPixelsNum(), INT32_MIN); + for (int m = 0; m < x.GetModulesNum(); m++) + TransferModuleAdjustMultipixels(image.data(), raw.data() + m * RAW_MODULE_SIZE, + x.GetModuleSlowDirectionStep(m), INT16_MIN, INT16_MAX, + x.GetModuleFastDirectionStep(m), + x.GetPixel0OfModuleConv(m)); + for (auto &v: image) { + if (v == INT16_MIN) + v = INT32_MIN; + else if (v == INT16_MAX) + v = INT32_MAX; + } + + std::vector squeezed(raw.size()); + SqueezeToRawGeometry(x, squeezed.data(), image.data(), INT32_MIN, INT32_MAX, + INT16_MIN, INT16_MAX); + CHECK(squeezed == raw); + + // A sum past the 16-bit range is saturated + image[x.GetPixel0OfModuleConv(0)] = 40000; + SqueezeToRawGeometry(x, squeezed.data(), image.data(), INT32_MIN, INT32_MAX, + INT16_MIN, INT16_MAX); + CHECK(squeezed[0] == INT16_MAX); } +} - std::vector squeezed(raw.size()); - SqueezeToRawGeometry(x, squeezed.data(), image.data(), INT32_MIN, INT32_MAX, - INT16_MIN, INT16_MAX); - CHECK(squeezed == raw); +TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry_4M","[RawToConvertedGeometry]") { + CheckSqueezeRoundTrip(DetJF(8, 2, 8, 36, true)); +} - // A sum past the 16-bit range is saturated - image[x.GetPixel0OfModuleConv(0)] = 40000; - SqueezeToRawGeometry(x, squeezed.data(), image.data(), INT32_MIN, INT32_MAX, - INT16_MIN, INT16_MAX); - CHECK(squeezed[0] == INT16_MAX); +TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry_9M","[RawToConvertedGeometry]") { + CheckSqueezeRoundTrip(DetJF(18, 3, 8, 36, true)); +} + +TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry_16_modules_turned","[RawToConvertedGeometry]") { + // 12 modules in rows of two, every other row read out the other way, and 4 standing modules beside them + using Direction = DetectorModuleGeometry::Direction; + std::vector modules; + for (int row = 0; row < 6; row++) + for (int col = 0; col < 2; col++) { + if (row % 2 == 0) + modules.emplace_back(col * 1038, row * 550, Direction::Xpos, Direction::Ypos); + else + modules.emplace_back(col * 1038 + 1029, row * 550 + 513, Direction::Xneg, Direction::Yneg); + } + for (int row = 0; row < 2; row++) + for (int col = 0; col < 2; col++) + modules.emplace_back(2076 + col * 522 + 513, row * 1038, Direction::Ypos, Direction::Xneg); + CheckSqueezeRoundTrip(DetJF(DetectorGeometryModular(modules))); } diff --git a/tools/jfjoch_fpga_stills_test.cpp b/tools/jfjoch_fpga_stills_test.cpp index fcd73d91c..2db0da20a 100644 --- a/tools/jfjoch_fpga_stills_test.cpp +++ b/tools/jfjoch_fpga_stills_test.cpp @@ -22,8 +22,9 @@ void print_usage(Logger &logger) { logger.Info("Usage ./jfjoch_fpga_stills_test {} "); - logger.Info("The dataset: JUNGFRAU modules, assembled with 8/36 pixel gaps. Its first images are loaded into"); - logger.Info("the card's internal generator, which cycles through them: images repeat, so throughput, latency"); + logger.Info("The dataset: JUNGFRAU modules, as the file states them (module groups, else the gaps of its mask)."); + logger.Info("Its first images - per card, as many as its generator memory holds of that card's modules - are"); + logger.Info("loaded into the cards' internal generators, which cycle through them: images repeat, so throughput, latency"); logger.Info("and indexing rate are meaningful, the merge and its statistics are not. See docs/TOOLS.md."); logger.Info("Options:"); logger.Info(" -v Verbose"); @@ -36,15 +37,44 @@ void print_usage(Logger &logger) { logger.Info(" -t Indexing thread pool size (default: 4)"); logger.Info(" -p FPGAImagePuller slots (default 128)"); logger.Info(" -F JFJochReceiverFPGA's own live stills instead of FPGAImagePuller"); - logger.Info(" -K{} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 32 MiB)"); + logger.Info(" -K{} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB ="); + logger.Info(" 128 module frames)"); } -// The images the internal generator holds for this run: what the card memory holds, at most what the run needs +// A module the file gives the readout of keeps it; one it does not (taken from the gaps of its mask, +x +y) is read +// out from the top-left corner of its box, along its 1030-pixel side. As the multipixels are symmetric in a module, +// which corner it is does not change the frames the card returns. +std::optional GeometryFromModules(const DetectorModules &modules) { + using Direction = DetectorModuleGeometry::Direction; + auto along_x = [](Direction d) { return (d == Direction::Xpos) || (d == Direction::Xneg); }; + + std::vector ret; + for (const auto &m: modules.Get()) { + if ((m.width == CONVERTED_MODULE_COLS) && (m.height == CONVERTED_MODULE_LINES) && along_x(m.fast)) + ret.emplace_back(m.ReadoutX0(), m.ReadoutY0(), m.fast, m.slow); + else if ((m.width == CONVERTED_MODULE_LINES) && (m.height == CONVERTED_MODULE_COLS) && !along_x(m.fast)) + ret.emplace_back(m.ReadoutX0(), m.ReadoutY0(), m.fast, m.slow); + else if ((m.width == CONVERTED_MODULE_LINES) && (m.height == CONVERTED_MODULE_COLS)) + ret.emplace_back(m.x0, m.y0, Direction::Ypos, Direction::Xpos); + else + return {}; + } + if (ret.empty()) + return {}; + return DetectorGeometryModular(ret); +} + +// The images the internal generator holds for this run: what the card memory and the firmware hold (not the broker's +// limit), at most what the run needs int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExperiment &x, int64_t file_images) { int64_t ret = std::min(file_images, x.GetImageNum()); - ret = std::min(ret, 64); // DetectorSettings::InternalGeneratorImages - for (int i = 0; i < x.GetDataStreamsNum(); i++) - ret = std::min(ret, (aq_devices[i].GetInternalGeneratorModuleFrames() - 1) / x.GetModulesNum(i)); + ret = std::min(ret, FPGA_INTERNAL_GENERATOR_MAX_IMAGES); + for (int i = 0; i < x.GetDataStreamsNum(); i++) { + // CheckInternalGeneratorFrame leaves the last module frame of the card memory free + const int64_t module_frames = std::min(aq_devices[i].GetInternalGeneratorModuleFrames() - 1, + FPGA_INTERNAL_GENERATOR_MAX_MODULE_FRAMES); + ret = std::min(ret, module_frames / x.GetModulesNum(i)); + } return ret; } @@ -59,16 +89,17 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s const auto dataset = reader.GetDataset(); const auto &file_x = dataset->experiment; - // JUNGFRAU modules (1030 x 514 assembled) with the gaps of DetectorGeometryModular as the PSI detectors have them - const int64_t width = file_x.GetXPixelsNum(), height = file_x.GetYPixelsNum(); - const int64_t module_x = CONVERTED_MODULE_COLS + 8, module_y = CONVERTED_MODULE_LINES + 36; - const int64_t cols = (width + 8) / module_x, rows = (height + 36) / module_y; - if ((cols * module_x - 8 != width) || (rows * module_y - 36 != height)) { - logger.Error("Image {} x {} is not a JUNGFRAU module layout with 8/36 pixel gaps", width, height); + // The modules as the file states them (module groups, else the gaps of its mask) + const auto geometry = GeometryFromModules(file_x.GetModules()); + if (!geometry || (geometry->GetWidth(true) != file_x.GetXPixelsNum()) + || (geometry->GetHeight(true) != file_x.GetYPixelsNum())) { + logger.Error("The file does not state JUNGFRAU modules (1030 x 514) that make up its {} x {} image", + file_x.GetXPixelsNum(), file_x.GetYPixelsNum()); return false; } - DiffractionExperiment x(DetJF(cols * rows, cols, 8, 36, true)); + DiffractionExperiment x(DetJF(geometry.value())); + x.Modules(file_x.GetModules()); x.ImagesPerTrigger(nimages.value_or(reader.GetNumberOfImages())).NumTriggers(1).PedestalG0Frames(0) .UseInternalPacketGenerator(true).DataStreams(nstreams); // The card returns the frames as loaded; the host assembles them as signed 16-bit photon counts @@ -86,14 +117,22 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s logger.Error("The card memory cannot hold one image of {} modules", x.GetModulesNum()); return false; } - x.InternalPacketGeneratorImages(generator_images); + x.InternalPacketGeneratorImages(generator_images, FPGA_INTERNAL_GENERATOR_MAX_IMAGES); PixelMask mask(x); if (dataset->pixel_mask->GetMask().size() == x.GetPixelsNum()) mask.LoadUserMask(x, dataset->pixel_mask->GetMask()); - logger.Info("Dataset {}: {} images of {} modules ({} x {}), {} distinct ones in the generator, {} in the run", - file, reader.GetNumberOfImages(), x.GetModulesNum(), cols, rows, generator_images, x.GetImageNum()); + logger.Info("Dataset {}: {} images of {} modules, {} distinct ones in the generator, {} in the run", + file, reader.GetNumberOfImages(), x.GetModulesNum(), generator_images, x.GetImageNum()); + for (int i = 0; i < x.GetDataStreamsNum(); i++) + logger.Info(" card {}: modules {}-{}, generator memory {} module frames, {} used", i, + x.GetFirstModuleOfDataStream(i), x.GetFirstModuleOfDataStream(i) + x.GetModulesNum(i) - 1, + aq_devices[i].GetInternalGeneratorModuleFrames(), x.GetModulesNum(i) * generator_images); + for (int m = 0; m < x.GetModulesNum(); m++) + logger.Debug(" module {}: pixel 0 at ({}, {}), fast step {}, slow step {}", m, + x.GetPixel0OfModuleConv(m) % x.GetXPixelsNum(), x.GetPixel0OfModuleConv(m) / x.GetXPixelsNum(), + x.GetModuleFastDirectionStep(m), x.GetModuleSlowDirectionStep(m)); logger.Info("Mode: {}, rugnux live stills", fpga_receiver ? "JFJochReceiverFPGA" : fmt::format("FPGAImagePuller ({} slots)", puller_slots)); @@ -237,8 +276,7 @@ int main(int argc, char **argv) { fpga_receiver = true; break; case 'K': - mock_card_MiB = (optarg != nullptr) ? atol(optarg) - : MockAcquisitionDevice::DefaultCardMemoryBytes / (1024 * 1024); + mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64; break; default: print_usage(logger);