jfjoch_process: Remove mutexes for reader and plots to improve performance (result not encouraging so far)
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 10m37s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 12m13s
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 14m2s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 15m2s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 15m7s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 15m19s
Build Packages / build:rpm (rocky8) (push) Successful in 11m17s
Build Packages / build:rpm (rocky9) (push) Successful in 10m25s
Build Packages / Generate python client (push) Successful in 34s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 10m10s
Build Packages / Build documentation (push) Successful in 58s
Build Packages / Create release (push) Has been skipped
Build Packages / XDS test (neggia plugin) (push) Successful in 8m53s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 11m46s
Build Packages / XDS test (durin plugin) (push) Successful in 10m50s
Build Packages / XDS test (JFJoch plugin) (push) Successful in 10m40s
Build Packages / DIALS test (push) Successful in 14m16s
Build Packages / build:rpm (rocky8_nocuda) (pull_request) Successful in 9m11s
Build Packages / build:rpm (ubuntu2404_nocuda) (pull_request) Successful in 11m3s
Build Packages / build:rpm (ubuntu2204_nocuda) (pull_request) Successful in 11m27s
Build Packages / build:rpm (rocky9_nocuda) (pull_request) Successful in 12m59s
Build Packages / build:rpm (rocky8_sls9) (pull_request) Successful in 13m45s
Build Packages / build:rpm (rocky9_sls9) (pull_request) Successful in 14m41s
Build Packages / build:rpm (rocky8) (pull_request) Successful in 11m45s
Build Packages / build:rpm (rocky9) (pull_request) Successful in 11m0s
Build Packages / Generate python client (pull_request) Successful in 54s
Build Packages / Build documentation (pull_request) Successful in 1m17s
Build Packages / build:rpm (ubuntu2404) (pull_request) Successful in 10m53s
Build Packages / Create release (pull_request) Has been skipped
Build Packages / build:rpm (ubuntu2204) (pull_request) Successful in 11m45s
Build Packages / XDS test (durin plugin) (pull_request) Successful in 9m52s
Build Packages / XDS test (neggia plugin) (pull_request) Successful in 9m14s
Build Packages / XDS test (JFJoch plugin) (pull_request) Successful in 9m34s
Build Packages / DIALS test (pull_request) Successful in 13m13s
Build Packages / Unit tests (push) Successful in 1h2m8s
Build Packages / Unit tests (pull_request) Successful in 54m32s
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 10m37s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 12m13s
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 14m2s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 15m2s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 15m7s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 15m19s
Build Packages / build:rpm (rocky8) (push) Successful in 11m17s
Build Packages / build:rpm (rocky9) (push) Successful in 10m25s
Build Packages / Generate python client (push) Successful in 34s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 10m10s
Build Packages / Build documentation (push) Successful in 58s
Build Packages / Create release (push) Has been skipped
Build Packages / XDS test (neggia plugin) (push) Successful in 8m53s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 11m46s
Build Packages / XDS test (durin plugin) (push) Successful in 10m50s
Build Packages / XDS test (JFJoch plugin) (push) Successful in 10m40s
Build Packages / DIALS test (push) Successful in 14m16s
Build Packages / build:rpm (rocky8_nocuda) (pull_request) Successful in 9m11s
Build Packages / build:rpm (ubuntu2404_nocuda) (pull_request) Successful in 11m3s
Build Packages / build:rpm (ubuntu2204_nocuda) (pull_request) Successful in 11m27s
Build Packages / build:rpm (rocky9_nocuda) (pull_request) Successful in 12m59s
Build Packages / build:rpm (rocky8_sls9) (pull_request) Successful in 13m45s
Build Packages / build:rpm (rocky9_sls9) (pull_request) Successful in 14m41s
Build Packages / build:rpm (rocky8) (pull_request) Successful in 11m45s
Build Packages / build:rpm (rocky9) (pull_request) Successful in 11m0s
Build Packages / Generate python client (pull_request) Successful in 54s
Build Packages / Build documentation (pull_request) Successful in 1m17s
Build Packages / build:rpm (ubuntu2404) (pull_request) Successful in 10m53s
Build Packages / Create release (pull_request) Has been skipped
Build Packages / build:rpm (ubuntu2204) (pull_request) Successful in 11m45s
Build Packages / XDS test (durin plugin) (pull_request) Successful in 9m52s
Build Packages / XDS test (neggia plugin) (pull_request) Successful in 9m14s
Build Packages / XDS test (JFJoch plugin) (pull_request) Successful in 9m34s
Build Packages / DIALS test (pull_request) Successful in 13m13s
Build Packages / Unit tests (push) Successful in 1h2m8s
Build Packages / Unit tests (pull_request) Successful in 54m32s
This commit is contained in:
+21
-37
@@ -389,9 +389,6 @@ int main(int argc, char **argv) {
|
||||
logger.Info("Starting analysis of {} images (range {}-{}) using {} threads",
|
||||
images_to_process, start_image, end_image, nthreads);
|
||||
|
||||
std::mutex reader_mutex;
|
||||
std::mutex plots_mutex; // Protect shared plot/stats updates if they aren't thread-safe
|
||||
|
||||
std::atomic<int> processed_count = 0;
|
||||
std::atomic<uint64_t> total_uncompressed_bytes = 0;
|
||||
std::atomic<uint64_t> max_image_number_sent = 0;
|
||||
@@ -426,14 +423,12 @@ int main(int argc, char **argv) {
|
||||
if (image_idx >= end_image) break;
|
||||
|
||||
// Load Image
|
||||
std::shared_ptr<JFJochReaderImage> img; {
|
||||
std::lock_guard<std::mutex> lock(reader_mutex);
|
||||
try {
|
||||
img = reader.LoadImage(image_idx);
|
||||
} catch (const std::exception &e) {
|
||||
logger.Error("Failed to load image {}: {}", image_idx, e.what());
|
||||
continue;
|
||||
}
|
||||
std::shared_ptr<JFJochReaderImage> img;
|
||||
try {
|
||||
img = reader.LoadImage(image_idx);
|
||||
} catch (const std::exception &e) {
|
||||
logger.Error("Failed to load image {}: {}", image_idx, e.what());
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!img) continue;
|
||||
@@ -462,32 +457,25 @@ int main(int argc, char **argv) {
|
||||
auto image_end_time = std::chrono::high_resolution_clock::now();
|
||||
std::chrono::duration<float> image_duration = image_end_time - image_start_time;
|
||||
|
||||
auto size = compressor.Compress(compressed_buffer.data(),
|
||||
img->Image().data(),
|
||||
experiment.GetPixelsNum(),
|
||||
sizeof(int32_t));
|
||||
|
||||
msg.image = CompressedImage(compressed_buffer.data(),
|
||||
size, experiment.GetXPixelsNum(),
|
||||
experiment.GetYPixelsNum(),
|
||||
CompressedImageMode::Int32,
|
||||
experiment.GetCompressionAlgorithm());
|
||||
|
||||
// Mimic DataMessage stats from Receiver
|
||||
msg.processing_time_s = image_duration.count();
|
||||
msg.original_number = msg.number;
|
||||
msg.run_number = experiment.GetRunNumber();
|
||||
msg.run_name = experiment.GetRunName();
|
||||
// msg.receiver_free_send_buf <- Not relevant for file analysis
|
||||
// msg.receiver_aq_dev_delay <- Not relevant for file analysis
|
||||
|
||||
// Update Plots/Stats (Thread safe update needed)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(plots_mutex);
|
||||
plots.Add(msg, profile);
|
||||
// Write Result
|
||||
if (writer)
|
||||
writer->Write(msg);
|
||||
plots.Add(msg, profile);
|
||||
// Write Result
|
||||
if (writer) {
|
||||
auto size = compressor.Compress(compressed_buffer.data(),
|
||||
img->Image().data(),
|
||||
experiment.GetPixelsNum(),
|
||||
sizeof(int32_t));
|
||||
|
||||
msg.image = CompressedImage(compressed_buffer.data(),
|
||||
size, experiment.GetXPixelsNum(),
|
||||
experiment.GetYPixelsNum(),
|
||||
CompressedImageMode::Int32,
|
||||
experiment.GetCompressionAlgorithm());
|
||||
writer->Write(msg);
|
||||
}
|
||||
|
||||
// Update max sent tracking
|
||||
@@ -501,11 +489,7 @@ int main(int argc, char **argv) {
|
||||
|
||||
// Progress log
|
||||
if (current_idx_offset > 0 && current_idx_offset % 100 == 0) {
|
||||
std::optional<float> indexing_rate;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(plots_mutex);
|
||||
indexing_rate = plots.GetIndexingRate();
|
||||
}
|
||||
std::optional<float> indexing_rate = plots.GetIndexingRate();
|
||||
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const double elapsed_s = std::chrono::duration<double>(now - start_time).count();
|
||||
|
||||
Reference in New Issue
Block a user