// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include #include #include #include #include "JFJochImageReadingWorker.h" #include "../image_analysis/geom_refinement/AssignSpotsToRings.h" #include "../image_analysis/spot_finding/StrongPixelSet.h" #include "../image_analysis/spot_finding/SpotUtils.h" #include "../image_analysis/spot_finding/ImageSpotFinder.h" #include #include #include #include "../preview/JFJochTIFF.h" namespace { enum class PreflightResult { Ok, NotYetVisible, // ENOENT, ESTALE, etc. PermissionDenied, // EACCES, EPERM IsDirectory, OtherError }; PreflightResult preflight_open_ro(const QString& filename, std::string& reason_out) { reason_out.clear(); const QByteArray path = filename.toLocal8Bit(); errno = 0; const int fd = ::open(path.constData(), O_RDONLY | O_CLOEXEC); if (fd >= 0) { struct stat st; if (fstat(fd, &st) == 0 && S_ISDIR(st.st_mode)) { ::close(fd); reason_out = "Path is a directory"; return PreflightResult::IsDirectory; } ::close(fd); return PreflightResult::Ok; } const int e = errno; reason_out = fmt::format("{} (errno={} {})", std::strerror(e), e, std::strerror(e)); // NFS can transiently report missing/stale entries. if (e == ENOENT || e == ESTALE || e == EIO || e == ETIMEDOUT || e == ENOTCONN) return PreflightResult::NotYetVisible; if (e == EACCES || e == EPERM) return PreflightResult::PermissionDenied; if (e == EISDIR) return PreflightResult::IsDirectory; return PreflightResult::OtherError; } } JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &settings, const DiffractionExperiment &experiment, QObject *parent) : QObject(parent), indexing_settings(experiment.GetIndexingSettings()), azint_settings(experiment.GetAzimuthalIntegrationSettings()) { spot_finding_settings = settings;; indexing = std::make_unique(indexing_settings); http_reader.Experiment(experiment); file_reader.Experiment(experiment); autoload_timer = new QTimer(this); autoload_timer->setInterval(autoload_interval); connect(autoload_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::AutoLoadTimerExpired); file_open_retry_timer = new QTimer(this); file_open_retry_timer->setSingleShot(true); connect(file_open_retry_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::FileOpenRetryTimerExpired); } void JFJochImageReadingWorker::ResetFileOpenRetry_i() { // Assumes m locked! if (file_open_retry_timer) file_open_retry_timer->stop(); // Signal UI to close the dialog if we were active if (file_open_retry_active) emit fileLoadRetryStatus(false, ""); file_open_retry_active = false; file_open_retry_warned = false; file_open_retry_attempts = 0; file_open_retry_delay_ms = 50; file_open_retry_elapsed.invalidate(); pending_load = {}; } void JFJochImageReadingWorker::ScheduleFileOpenRetry_i(const QString& reason) { // Assumes m locked! if (!file_open_retry_active) { file_open_retry_active = true; file_open_retry_warned = false; file_open_retry_attempts = 0; file_open_retry_delay_ms = 50; file_open_retry_elapsed.restart(); } if (!file_open_retry_warned) { file_open_retry_warned = true; logger.Warning(fmt::format( "File '{}' not available yet (GPFS/NFS). Retrying with back-off up to 10 s. Reason: {}", pending_load.filename.toStdString(), reason.toStdString())); // Signal UI to show the dialog emit fileLoadRetryStatus(true, fmt::format("Waiting for file {} to appear on disk...", pending_load.filename.toStdString()).c_str()); } else { logger.Debug(fmt::format( "Retry pending for file '{}'. Reason: {}", pending_load.filename.toStdString(), reason.toStdString())); } if (file_open_retry_elapsed.isValid() && file_open_retry_elapsed.elapsed() >= 10'000) { std::string msg = fmt::format( "Timed out waiting for file '{}' after 10 s ({} attempt(s))", pending_load.filename.toStdString(), file_open_retry_attempts); logger.Error(msg); // Reset first (closes the progress dialog) ResetFileOpenRetry_i(); // Then show the error dialog emit fileLoadError("File Open Timeout", QString::fromStdString(msg)); return; } const int delay = file_open_retry_delay_ms; file_open_retry_delay_ms = std::min(file_open_retry_delay_ms * 2, file_open_retry_delay_max_ms); if (file_open_retry_timer) file_open_retry_timer->start(delay); } void JFJochImageReadingWorker::FileOpenRetryTimerExpired() { PendingLoadRequest req; QMutexLocker ul(&m); if (!file_open_retry_active) return; req = pending_load; // Re-trigger LoadFile, but keep retry=true so we stay in the retry loop. LoadFile_i(req.filename, req.image_number, req.summation, true); } void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_number, qint64 summation, bool retry) { try { std::shared_ptr dataset; auto start = std::chrono::high_resolution_clock::now(); if (filename.startsWith("http://")) { http_mode = true; http_reader.ReadURL(filename.toStdString()); total_images = http_reader.GetNumberOfImages(); dataset = http_reader.GetDataset(); if (image_number < 0) setAutoLoadMode_i(AutoloadMode::HTTPSync); else setAutoLoadMode_i(AutoloadMode::None); } else { http_mode = false; if (retry) { pending_load.filename = filename; pending_load.image_number = image_number; pending_load.summation = summation; std::string reason; const PreflightResult pr = preflight_open_ro(filename, reason); switch (pr) { case PreflightResult::Ok: break; case PreflightResult::NotYetVisible: ScheduleFileOpenRetry_i(QString::fromStdString(reason)); return; // IMPORTANT: do not try to open the file yet case PreflightResult::PermissionDenied: logger.Error(fmt::format( "Permission denied opening '{}' (read-only preflight failed: {}). Not retrying.", filename.toStdString(), reason)); emit fileLoadError("Permission Denied", QString::fromStdString(reason)); ResetFileOpenRetry_i(); return; case PreflightResult::IsDirectory: logger.Error(fmt::format( "Error opening '{}': {}. Not retrying.", filename.toStdString(), reason)); emit fileLoadError("Cannot open directory", QString::fromStdString(reason)); ResetFileOpenRetry_i(); return; case PreflightResult::OtherError: logger.Error(fmt::format( "Other error '{}' (read-only preflight failed: {}). Not retrying.", filename.toStdString(), reason)); emit fileLoadError("File Open Error", QString::fromStdString(reason)); ResetFileOpenRetry_i(); return; } // At this point we will attempt the real open. file_open_retry_attempts++; } file_reader.ReadFile(filename.toStdString()); total_images = file_reader.GetNumberOfImages(); dataset = file_reader.GetDataset(); setAutoLoadMode_i(AutoloadMode::None); if (retry && file_open_retry_active) { logger.Info(fmt::format( "File '{}' opened after {} attempt(s), waited {} ms", filename.toStdString(), file_open_retry_attempts, file_open_retry_elapsed.isValid() ? file_open_retry_elapsed.elapsed() : 0)); } ResetFileOpenRetry_i(); } current_image.reset(); current_summation = 1; current_file = filename; if (dataset) { curr_experiment = dataset->experiment; curr_experiment.ImportIndexingSettings(indexing_settings); curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings); UpdateAzint_i(dataset.get()); } emit datasetLoaded(dataset); auto end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end - start).count(); logger.Info(fmt::format("Loaded file {} in {} ms", filename.toStdString(), duration)); LoadImage_i(image_number, summation); } catch (std::exception &e) { logger.Error(fmt::format("Error loading file {} {}", filename.toStdString(), e.what())); emit fileLoadError("File Load Error", QString::fromStdString(e.what())); ResetFileOpenRetry_i(); emit datasetLoaded({}); emit imageLoaded({}); } } void JFJochImageReadingWorker::LoadFile(const QString &filename, qint64 image_number, qint64 summation, bool retry) { QMutexLocker ul(&m); ResetFileOpenRetry_i(); LoadFile_i(filename, image_number, summation, retry); } void JFJochImageReadingWorker::CloseFile() { QMutexLocker ul(&m); ResetFileOpenRetry_i(); if (http_mode) http_reader.Close(); else file_reader.Close(); current_image_ptr.reset(); current_image.reset(); current_summation = 1; total_images = 0; current_file = ""; emit imageLoaded({}); emit datasetLoaded({}); } void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation) { QMutexLocker ul(&m); setAutoLoadMode_i(AutoloadMode::None); if ((image_number == current_image) && (current_summation == summation)) return; LoadImage_i(image_number, summation); } void JFJochImageReadingWorker::UpdateAzint_i(const JFJochReaderDataset *dataset) { if (dataset) { azint_mapping = std::make_unique(curr_experiment, dataset->pixel_mask); index_and_refine = std::make_unique(curr_experiment, indexing.get()); image_analysis = std::make_unique(curr_experiment, *azint_mapping, dataset->pixel_mask, *index_and_refine.get()); } } void JFJochImageReadingWorker::LoadImage_i(int64_t image_number, int64_t summation) { // Assumes m locked! try { if (summation <= 0) return; std::vector image; auto start = std::chrono::high_resolution_clock::now(); if (http_mode) { if (image_number < 0 && summation != 1) return; const auto tmp_image_ptr = http_reader.LoadImage(image_number, summation); if (image_number < 0 && tmp_image_ptr == nullptr) { logger.Debug("No change in online buffer, not updating viewer"); return; // Do nothing, since there is no update in the file } current_image_ptr = tmp_image_ptr; total_images = http_reader.GetNumberOfImages(); emit datasetLoaded(http_reader.GetDataset()); } else { if (image_number < 0 || image_number + summation > total_images) return; current_image_ptr = file_reader.LoadImage(image_number, summation); } if (!current_image_ptr) { emit imageLoaded({}); return; } current_image = current_image_ptr->ImageData().number; current_summation = summation; auto end = std::chrono::high_resolution_clock::now(); if (auto_reanalyze) ReanalyzeImage_i(); auto end_analysis = std::chrono::high_resolution_clock::now(); auto duration_1 = std::chrono::duration_cast(end - start).count(); auto duration_2 = std::chrono::duration_cast(end_analysis - end).count(); // Adapt autoload interval from moving average of (load + analysis) time if (autoload_mode != AutoloadMode::None) { const int total_ms = static_cast(duration_1 + duration_2); autoload_ms_ma.Add(static_cast(total_ms)); const auto avg_ms = autoload_ms_ma.Read(); if (avg_ms.has_value()) { int proposed_ms = static_cast(avg_ms.value() * autoload_safety_factor); proposed_ms = std::clamp(proposed_ms, autoload_interval_min_ms, autoload_interval_max_ms); if (proposed_ms != autoload_interval) { autoload_interval = proposed_ms; autoload_timer->setInterval(autoload_interval); } } } logger.Info("Loaded image {} in {}/{} ms Autoload timer set to {} ms", image_number, duration_1, duration_2, autoload_interval); emit imageNumberChanged(total_images, current_image.value()); emit imageLoaded(current_image_ptr); } catch (std::exception &e) { logger.Error("Error loading image {}: {}", image_number, e.what()); } } void JFJochImageReadingWorker::SetROIBox(QRect box) { QMutexLocker ul(&m); if (box.width() * box.height() == 0) roi.reset(); roi = std::make_unique("roi1", box.left(), box.right(), box.bottom(), box.top()); } void JFJochImageReadingWorker::SetROICircle(double x, double y, double radius) { QMutexLocker ul(&m); if (radius <= 0) roi.reset(); else roi = std::make_unique("roi1", x, y, radius); } void JFJochImageReadingWorker::UpdateDataset_i(const std::optional &experiment) { if (!current_image_ptr) return; std::shared_ptr dataset; if (http_mode) { if (experiment) http_reader.UpdateGeomMetadata(experiment.value()); dataset = http_reader.GetDataset(); } else { if (experiment) file_reader.UpdateGeomMetadata(experiment.value()); dataset = file_reader.GetDataset(); } if (!dataset) { logger.Error("UpdateDataset_i: dataset is null (http_mode={}) - skipping update to avoid crash", http_mode); return; } curr_experiment = dataset->experiment; curr_experiment.ImportIndexingSettings(indexing_settings); curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings); UpdateAzint_i(dataset.get()); emit datasetLoaded(dataset); current_image_ptr = std::make_shared(current_image_ptr->ImageData(), dataset); if (auto_reanalyze) ReanalyzeImage_i(); emit imageLoaded(current_image_ptr); } void JFJochImageReadingWorker::UpdateDataset(const DiffractionExperiment &experiment) { QMutexLocker ul(&m); UpdateDataset_i(experiment); } void JFJochImageReadingWorker::ReanalyzeImage_i() { if (!current_image_ptr || !azint_mapping || !image_analysis) return; auto start_time = std::chrono::high_resolution_clock::now(); auto new_image = std::make_shared(*current_image_ptr); auto new_image_dataset = new_image->CreateMutableDataset(); new_image_dataset->experiment.ImportIndexingSettings(indexing_settings); new_image_dataset->experiment.ImportAzimuthalIntegrationSettings(azint_settings); new_image_dataset->az_int_bin_to_phi = azint_mapping->GetBinToPhi(); new_image_dataset->az_int_bin_to_q = azint_mapping->GetBinToQ(); new_image_dataset->azimuthal_bins = azint_mapping->GetAzimuthalBinCount(); new_image_dataset->q_bins = azint_mapping->GetQBinCount(); std::vector buffer; AzimuthalIntegrationProfile azint_profile(*azint_mapping); image_analysis->Analyze(new_image->ImageData(), buffer, azint_profile, spot_finding_settings); current_image_ptr = new_image; auto end_time = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end_time - start_time); logger.Info("Analysis of image in {} ms", duration.count()); } void JFJochImageReadingWorker::Analyze() { QMutexLocker locker(&m); ReanalyzeImage_i(); emit imageLoaded(current_image_ptr); } void JFJochImageReadingWorker::FindCenter(const UnitCell& calibrant, bool guess) { QMutexLocker locker(&m); if (!current_image_ptr) return; logger.Info("Finding center"); DiffractionGeometry geom = current_image_ptr->Dataset().experiment.GetDiffractionGeometry(); try { if (guess) GuessGeometry(geom, current_image_ptr->ImageData().spots, calibrant); else OptimizeGeometry(geom, current_image_ptr->ImageData().spots, calibrant); } catch (const JFJochException &e) { logger.ErrorException(e); return; } logger.Info("Geometry found X: {} pxl Y: {} pxl Dist: {} mm", geom.GetBeamX_pxl(), geom.GetBeamY_pxl(), geom.GetDetectorDistance_mm()); DiffractionExperiment new_experiment = current_image_ptr->Dataset().experiment; new_experiment.BeamX_pxl(geom.GetBeamX_pxl()).BeamY_pxl(geom.GetBeamY_pxl()) .DetectorDistance_mm(geom.GetDetectorDistance_mm()) .PoniRot1_rad(geom.GetPoniRot1_rad()) .PoniRot2_rad(geom.GetPoniRot2_rad()) .PoniRot3_rad(geom.GetPoniRot3_rad()); UpdateDataset_i(new_experiment); std::vector ring_Q = CalculateXtalRings(calibrant); QVector rings; for (int i = 0; i < 15 && i < ring_Q.size(); i++) { rings.push_back(2 * M_PI / ring_Q[i]); } emit setRings(rings); } void JFJochImageReadingWorker::UpdateSpotFindingSettings(const SpotFindingSettings &settings, const IndexingSettings &indexing, int64_t max_spots) { QMutexLocker locker(&m); spot_finding_settings = settings; indexing_settings.Tolerance(indexing.GetTolerance()); indexing_settings.ViableCellMinSpots(indexing.GetViableCellMinSpots()); indexing_settings.IndexIceRings(indexing.GetIndexIceRings()); indexing_settings.UnitCellDistTolerance(indexing.GetUnitCellDistTolerance()); curr_experiment.ImportIndexingSettings(indexing_settings); curr_experiment.MaxSpotCount(max_spots); if (auto_reanalyze) { ReanalyzeImage_i(); emit imageLoaded(current_image_ptr); } } void JFJochImageReadingWorker::ReanalyzeImages(bool input) { QMutexLocker locker(&m); auto_reanalyze = input; if (auto_reanalyze) { ReanalyzeImage_i(); emit imageLoaded(current_image_ptr); } } void JFJochImageReadingWorker::UpdateAzintSettings(const AzimuthalIntegrationSettings &settings) { QMutexLocker locker(&m); azint_settings = settings; UpdateDataset_i(std::nullopt); } void JFJochImageReadingWorker::UpdateUserMask_i(const std::vector &mask) { std::shared_ptr dataset; if (http_mode) { http_reader.UpdateUserMask(mask); dataset = http_reader.GetDataset(); } else { file_reader.UpdateUserMask(mask); dataset = file_reader.GetDataset(); } if (!dataset) { logger.Error("UpdateUserMask_i: dataset is null (http_mode={}) - skipping update to avoid crash", http_mode); return; } UpdateAzint_i(dataset.get()); emit datasetLoaded(dataset); current_image_ptr = std::make_shared(current_image_ptr->ImageData(), dataset); if (current_image.has_value()) LoadImage_i(current_image.value(), current_summation); } void JFJochImageReadingWorker::AddROIToUserMask() { QMutexLocker locker(&m); if (!roi || !current_image_ptr) return; auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask(); int64_t width = current_image_ptr->Dataset().experiment.GetXPixelsNum(); int64_t height = current_image_ptr->Dataset().experiment.GetYPixelsNum(); const auto res = azint_mapping->Resolution(); for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { if (roi->CheckROI(x, y, 0)) user_mask[x + y * width] = 1; } } UpdateUserMask_i(user_mask); } void JFJochImageReadingWorker::SubtractROIFromUserMask() { QMutexLocker locker(&m); if (!roi || !current_image_ptr) return; auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask(); int64_t width = current_image_ptr->Dataset().experiment.GetXPixelsNum(); int64_t height = current_image_ptr->Dataset().experiment.GetYPixelsNum(); for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { if (roi->CheckROI(x, y, 0)) user_mask[x + y * width] = 0; } } UpdateUserMask_i(user_mask); } void JFJochImageReadingWorker::ClearUserMask() { QMutexLocker locker(&m); if (!roi || !current_image_ptr) return; auto user_mask = std::vector(current_image_ptr->Dataset().experiment.GetXPixelsNum() * current_image_ptr->Dataset().experiment.GetYPixelsNum(), 0); UpdateUserMask_i(user_mask); } void JFJochImageReadingWorker::SaveUserMaskTIFF(QString filename) { QMutexLocker locker(&m); if (!current_image_ptr) return; auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask(); CompressedImage mask_image(user_mask, current_image_ptr->Dataset().experiment.GetXPixelsNum(), current_image_ptr->Dataset().experiment.GetYPixelsNum()); WriteTIFFToFile(filename.toStdString(), mask_image); } void JFJochImageReadingWorker::UploadUserMask() { QMutexLocker locker(&m); if (!current_image_ptr) return; auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask(); if (http_mode) http_reader.UploadUserMask(user_mask); } void JFJochImageReadingWorker::LoadCalibration(QString dataset) { if (!http_mode) { auto tmp = std::make_shared(); try { tmp->image = file_reader.ReadCalibration(tmp->buffer, dataset.toStdString()); std::shared_ptr ctmp = tmp; emit simpleImageLoaded(ctmp); } catch (const std::exception &e) { logger.Info("Error loading calibration: {}", e.what()); } } else logger.Info("HTTP mode doesn't allow to read calibration (at the moment"); } void JFJochImageReadingWorker::AutoLoadTimerExpired() { QMutexLocker locker(&m); switch (autoload_mode) { case AutoloadMode::HTTPSync: if (http_mode) LoadImage_i(-1 , 1); break; case AutoloadMode::Movie: { if (total_images == 0 || !current_image) return; int64_t new_image = (current_image.value() + jump_value) % total_images; LoadImage_i(new_image, current_summation); break; } case AutoloadMode::None: break; } } void JFJochImageReadingWorker::setAutoLoadMode_i(AutoloadMode in_mode) { autoload_mode = in_mode; if (autoload_mode == AutoloadMode::None) autoload_timer->stop(); else autoload_timer->start(); emit autoloadChanged(autoload_mode); } void JFJochImageReadingWorker::setAutoLoadMode(AutoloadMode mode) { QMutexLocker ul(&m); switch (mode) { case AutoloadMode::HTTPSync: if (http_mode) setAutoLoadMode_i(mode); else setAutoLoadMode_i(AutoloadMode::None); break; case AutoloadMode::Movie: setAutoLoadMode_i(mode); break; case AutoloadMode::None: setAutoLoadMode_i(mode); break; } } void JFJochImageReadingWorker::setAutoLoadJump(int64_t val) { QMutexLocker ul(&m); if (val > 0) jump_value = val; }