added roi, noise and gain (#143)

- Moved definitions of Cluster_2x2 and Cluster_3x3 to it's own file
- Added optional members for ROI, noise_map and gain_map in ClusterFile

**API:**

After creating the ClusterFile the user can set one or all of: roi,
noise_map, gain_map

```python
f = ClusterFile(fname)
f.set_roi(roi) #aare.ROI
f.set_noise_map(noise_map) #numpy array
f.set_gain_map(gain_map) #numpy array
```

**When reading clusters they are evaluated in the order:**

1. If ROI is enabled check that the cluster is within the ROI
1. If noise_map is enabled check that the cluster meets one of the
conditions
    - Center pixel above noise
    - Highest 2x2 sum above 2x noise
    - 3x3 sum above 3x noise
1. If gain_map is set apply the gain map before returning the clusters
(not used for noise cut)

**Open questions:**
1. Check for out of bounds access in noise and gain map?

closes #139 
closes #135 
closes #90
This commit is contained in:
Erik Fröjdh
2025-04-01 14:31:25 +02:00
committed by GitHub
parent 6ad76f63c1
commit a42c0d645b
12 changed files with 400 additions and 345 deletions

View File

@ -31,6 +31,18 @@ ClusterFile::ClusterFile(const std::filesystem::path &fname, size_t chunk_size,
}
}
void ClusterFile::set_roi(ROI roi){
m_roi = roi;
}
void ClusterFile::set_noise_map(const NDView<int32_t, 2> noise_map){
m_noise_map = NDArray<int32_t, 2>(noise_map);
}
void ClusterFile::set_gain_map(const NDView<double, 2> gain_map){
m_gain_map = NDArray<double, 2>(gain_map);
}
ClusterFile::~ClusterFile() { close(); }
void ClusterFile::close() {
@ -48,14 +60,37 @@ void ClusterFile::write_frame(const ClusterVector<int32_t> &clusters) {
!(clusters.cluster_size_y() == 3)) {
throw std::runtime_error("Only 3x3 clusters are supported");
}
//First write the frame number - 4 bytes
int32_t frame_number = clusters.frame_number();
fwrite(&frame_number, sizeof(frame_number), 1, fp);
if(fwrite(&frame_number, sizeof(frame_number), 1, fp)!=1){
throw std::runtime_error(LOCATION + "Could not write frame number");
}
//Then write the number of clusters - 4 bytes
uint32_t n_clusters = clusters.size();
fwrite(&n_clusters, sizeof(n_clusters), 1, fp);
fwrite(clusters.data(), clusters.item_size(), clusters.size(), fp);
if(fwrite(&n_clusters, sizeof(n_clusters), 1, fp)!=1){
throw std::runtime_error(LOCATION + "Could not write number of clusters");
}
//Now write the clusters in the frame
if(fwrite(clusters.data(), clusters.item_size(), clusters.size(), fp)!=clusters.size()){
throw std::runtime_error(LOCATION + "Could not write clusters");
}
}
ClusterVector<int32_t> ClusterFile::read_clusters(size_t n_clusters) {
ClusterVector<int32_t> ClusterFile::read_clusters(size_t n_clusters){
if (m_mode != "r") {
throw std::runtime_error("File not opened for reading");
}
if (m_noise_map || m_roi){
return read_clusters_with_cut(n_clusters);
}else{
return read_clusters_without_cut(n_clusters);
}
}
ClusterVector<int32_t> ClusterFile::read_clusters_without_cut(size_t n_clusters) {
if (m_mode != "r") {
throw std::runtime_error("File not opened for reading");
}
@ -86,6 +121,7 @@ ClusterVector<int32_t> ClusterFile::read_clusters(size_t n_clusters) {
if (nph_read < n_clusters) {
// keep on reading frames and photons until reaching n_clusters
while (fread(&iframe, sizeof(iframe), 1, fp)) {
clusters.set_frame_number(iframe);
// read number of clusters in frame
if (fread(&nph, sizeof(nph), 1, fp)) {
if (nph > (n_clusters - nph_read))
@ -105,71 +141,112 @@ ClusterVector<int32_t> ClusterFile::read_clusters(size_t n_clusters) {
// Resize the vector to the number of clusters.
// No new allocation, only change bounds.
clusters.resize(nph_read);
if(m_gain_map)
clusters.apply_gain_map(m_gain_map->view());
return clusters;
}
ClusterVector<int32_t> ClusterFile::read_clusters(size_t n_clusters, ROI roi) {
if (m_mode != "r") {
throw std::runtime_error("File not opened for reading");
}
ClusterVector<int32_t> ClusterFile::read_clusters_with_cut(size_t n_clusters) {
ClusterVector<int32_t> clusters(3,3);
clusters.reserve(n_clusters);
Cluster3x3 tmp; //this would break if the cluster size changes
// if there are photons left from previous frame read them first
if (m_num_left) {
size_t nph_read = 0;
while(nph_read < m_num_left && clusters.size() < n_clusters){
fread(&tmp, sizeof(tmp), 1, fp);
nph_read++;
if(tmp.x >= roi.xmin && tmp.x <= roi.xmax && tmp.y >= roi.ymin && tmp.y <= roi.ymax){
clusters.push_back(tmp.x, tmp.y, reinterpret_cast<std::byte*>(tmp.data));
while(m_num_left && clusters.size() < n_clusters){
Cluster3x3 c = read_one_cluster();
if(is_selected(c)){
clusters.push_back(c.x, c.y, reinterpret_cast<std::byte*>(c.data));
}
}
m_num_left -= nph_read;
}
// we did not have enough clusters left in the previous frame
// keep on reading frames until reaching n_clusters
if (clusters.size() < n_clusters) {
// sanity check
if (m_num_left) {
throw std::runtime_error(LOCATION + "Entered second loop with clusters left\n");
}
// we did not have enough clusters left in the previous frame
// keep on reading frames until reaching n_clusters
int32_t frame_number = 0; // frame number needs to be 4 bytes!
while (fread(&frame_number, sizeof(frame_number), 1, fp)) {
uint32_t nph_in_frame = 0; //number of photons we can read until next frame number
size_t nph_read = 0; //number of photons read in this frame
if (fread(&nph_in_frame, sizeof(nph_in_frame), 1, fp)) {
if(frame_number != 1){
throw std::runtime_error("Frame number is not 1");
}
while(nph_read < nph_in_frame && clusters.size() < n_clusters){
fread(&tmp, sizeof(tmp), 1, fp);
nph_read++;
if(tmp.x >= roi.xmin && tmp.x <= roi.xmax && tmp.y >= roi.ymin && tmp.y <= roi.ymax){
clusters.push_back(tmp.x, tmp.y, reinterpret_cast<std::byte*>(tmp.data));
if (fread(&m_num_left, sizeof(m_num_left), 1, fp)) {
clusters.set_frame_number(frame_number); //cluster vector will hold the last frame number
while(m_num_left && clusters.size() < n_clusters){
Cluster3x3 c = read_one_cluster();
if(is_selected(c)){
clusters.push_back(c.x, c.y, reinterpret_cast<std::byte*>(c.data));
}
}
m_num_left = nph_in_frame - nph_read;
}
if (clusters.size() >= n_clusters){
// we have enough clusters, break out of the outer while loop
if (clusters.size() >= n_clusters)
break;
}
}
}
if(m_gain_map)
clusters.apply_gain_map(m_gain_map->view());
return clusters;
}
ClusterVector<int32_t> ClusterFile::read_frame() {
Cluster3x3 ClusterFile::read_one_cluster(){
Cluster3x3 c;
auto rc = fread(&c, sizeof(c), 1, fp);
if (rc != 1) {
throw std::runtime_error(LOCATION + "Could not read cluster");
}
--m_num_left;
return c;
}
ClusterVector<int32_t> ClusterFile::read_frame(){
if (m_mode != "r") {
throw std::runtime_error(LOCATION + "File not opened for reading");
}
if (m_noise_map || m_roi){
return read_frame_with_cut();
}else{
return read_frame_without_cut();
}
}
ClusterVector<int32_t> ClusterFile::read_frame_without_cut() {
if (m_mode != "r") {
throw std::runtime_error("File not opened for reading");
}
if (m_num_left) {
throw std::runtime_error(
"There are still photons left in the last frame");
}
int32_t frame_number;
if (fread(&frame_number, sizeof(frame_number), 1, fp) != 1) {
throw std::runtime_error(LOCATION + "Could not read frame number");
}
int32_t n_clusters; // Saved as 32bit integer in the cluster file
if (fread(&n_clusters, sizeof(n_clusters), 1, fp) != 1) {
throw std::runtime_error(LOCATION + "Could not read number of clusters");
}
ClusterVector<int32_t> clusters(3, 3, n_clusters);
clusters.set_frame_number(frame_number);
if (fread(clusters.data(), clusters.item_size(), n_clusters, fp) !=
static_cast<size_t>(n_clusters)) {
throw std::runtime_error(LOCATION + "Could not read clusters");
}
clusters.resize(n_clusters);
if (m_gain_map)
clusters.apply_gain_map(m_gain_map->view());
return clusters;
}
ClusterVector<int32_t> ClusterFile::read_frame_with_cut() {
if (m_mode != "r") {
throw std::runtime_error("File not opened for reading");
}
@ -182,149 +259,47 @@ ClusterVector<int32_t> ClusterFile::read_frame() {
throw std::runtime_error("Could not read frame number");
}
int32_t n_clusters; // Saved as 32bit integer in the cluster file
if (fread(&n_clusters, sizeof(n_clusters), 1, fp) != 1) {
if (fread(&m_num_left, sizeof(m_num_left), 1, fp) != 1) {
throw std::runtime_error("Could not read number of clusters");
}
// std::vector<Cluster3x3> clusters(n_clusters);
ClusterVector<int32_t> clusters(3, 3, n_clusters);
ClusterVector<int32_t> clusters(3, 3);
clusters.reserve(m_num_left);
clusters.set_frame_number(frame_number);
if (fread(clusters.data(), clusters.item_size(), n_clusters, fp) !=
static_cast<size_t>(n_clusters)) {
throw std::runtime_error("Could not read clusters");
while(m_num_left){
Cluster3x3 c = read_one_cluster();
if(is_selected(c)){
clusters.push_back(c.x, c.y, reinterpret_cast<std::byte*>(c.data));
}
}
clusters.resize(n_clusters);
if (m_gain_map)
clusters.apply_gain_map(m_gain_map->view());
return clusters;
}
// std::vector<Cluster3x3> ClusterFile::read_cluster_with_cut(size_t n_clusters,
// double *noise_map,
// int nx, int ny) {
// if (m_mode != "r") {
// throw std::runtime_error("File not opened for reading");
// }
// std::vector<Cluster3x3> clusters(n_clusters);
// // size_t read_clusters_with_cut(FILE *fp, size_t n_clusters, Cluster *buf,
// // uint32_t *n_left, double *noise_map, int
// // nx, int ny) {
// int iframe = 0;
// // uint32_t nph = *n_left;
// uint32_t nph = m_num_left;
// // uint32_t nn = *n_left;
// uint32_t nn = m_num_left;
// size_t nph_read = 0;
// int32_t t2max, tot1;
// int32_t tot3;
// // Cluster *ptr = buf;
// Cluster3x3 *ptr = clusters.data();
// int good = 1;
// double noise;
// // read photons left from previous frame
// if (noise_map)
// printf("Using noise map\n");
bool ClusterFile::is_selected(Cluster3x3 &cl) {
//Should fail fast
if (m_roi) {
if (!(m_roi->contains(cl.x, cl.y))) {
return false;
}
}
if (m_noise_map){
int32_t sum_1x1 = cl.data[4]; // central pixel
int32_t sum_2x2 = cl.sum_2x2(); // highest sum of 2x2 subclusters
int32_t sum_3x3 = cl.sum(); // sum of all pixels
// if (nph) {
// if (nph > n_clusters) {
// // if we have more photons left in the frame then photons to
// // read we read directly the requested number
// nn = n_clusters;
// } else {
// nn = nph;
// }
// for (size_t iph = 0; iph < nn; iph++) {
// // read photons 1 by 1
// size_t n_read =
// fread(reinterpret_cast<void *>(ptr), sizeof(Cluster3x3), 1, fp);
// if (n_read != 1) {
// clusters.resize(nph_read);
// return clusters;
// }
// // TODO! error handling on read
// good = 1;
// if (noise_map) {
// if (ptr->x >= 0 && ptr->x < nx && ptr->y >= 0 && ptr->y < ny) {
// tot1 = ptr->data[4];
// analyze_cluster(*ptr, &t2max, &tot3, NULL, NULL, NULL, NULL,
// NULL);
// noise = noise_map[ptr->y * nx + ptr->x];
// if (tot1 > noise || t2max > 2 * noise || tot3 > 3 * noise) {
// ;
// } else {
// good = 0;
// printf("%d %d %f %d %d %d\n", ptr->x, ptr->y, noise,
// tot1, t2max, tot3);
// }
// } else {
// printf("Bad pixel number %d %d\n", ptr->x, ptr->y);
// good = 0;
// }
// }
// if (good) {
// ptr++;
// nph_read++;
// }
// (m_num_left)--;
// if (nph_read >= n_clusters)
// break;
// }
// }
// if (nph_read < n_clusters) {
// // // keep on reading frames and photons until reaching
// // n_clusters
// while (fread(&iframe, sizeof(iframe), 1, fp)) {
// // // printf("%d\n",nph_read);
// if (fread(&nph, sizeof(nph), 1, fp)) {
// // // printf("** %d\n",nph);
// m_num_left = nph;
// for (size_t iph = 0; iph < nph; iph++) {
// // // read photons 1 by 1
// size_t n_read = fread(reinterpret_cast<void *>(ptr),
// sizeof(Cluster3x3), 1, fp);
// if (n_read != 1) {
// clusters.resize(nph_read);
// return clusters;
// // return nph_read;
// }
// good = 1;
// if (noise_map) {
// if (ptr->x >= 0 && ptr->x < nx && ptr->y >= 0 &&
// ptr->y < ny) {
// tot1 = ptr->data[4];
// analyze_cluster(*ptr, &t2max, &tot3, NULL, NULL,
// NULL, NULL, NULL);
// // noise = noise_map[ptr->y * nx + ptr->x];
// noise = noise_map[ptr->y + ny * ptr->x];
// if (tot1 > noise || t2max > 2 * noise ||
// tot3 > 3 * noise) {
// ;
// } else
// good = 0;
// } else {
// printf("Bad pixel number %d %d\n", ptr->x, ptr->y);
// good = 0;
// }
// }
// if (good) {
// ptr++;
// nph_read++;
// }
// (m_num_left)--;
// if (nph_read >= n_clusters)
// break;
// }
// }
// if (nph_read >= n_clusters)
// break;
// }
// }
// // printf("%d\n",nph_read);
// clusters.resize(nph_read);
// return clusters;
// }
auto noise = (*m_noise_map)(cl.y, cl.x); //TODO! check if this is correct
if (sum_1x1 <= noise || sum_2x2 <= 2 * noise || sum_3x3 <= 3 * noise) {
return false;
}
}
//we passed all checks
return true;
}
NDArray<double, 2> calculate_eta2(ClusterVector<int> &clusters) {
//TOTO! make work with 2x2 clusters
@ -419,111 +394,4 @@ Eta2 calculate_eta2(Cluster2x2 &cl) {
}
int analyze_cluster(Cluster3x3 &cl, int32_t *t2, int32_t *t3, char *quad,
double *eta2x, double *eta2y, double *eta3x,
double *eta3y) {
return analyze_data(cl.data, t2, t3, quad, eta2x, eta2y, eta3x, eta3y);
}
int analyze_data(int32_t *data, int32_t *t2, int32_t *t3, char *quad,
double *eta2x, double *eta2y, double *eta3x, double *eta3y) {
int ok = 1;
int32_t tot2[4];
int32_t t2max = 0;
char c = 0;
int32_t val, tot3;
tot3 = 0;
for (int i = 0; i < 4; i++)
tot2[i] = 0;
for (int ix = 0; ix < 3; ix++) {
for (int iy = 0; iy < 3; iy++) {
val = data[iy * 3 + ix];
// printf ("%d ",data[iy * 3 + ix]);
tot3 += val;
if (ix <= 1 && iy <= 1)
tot2[cBottomLeft] += val;
if (ix >= 1 && iy <= 1)
tot2[cBottomRight] += val;
if (ix <= 1 && iy >= 1)
tot2[cTopLeft] += val;
if (ix >= 1 && iy >= 1)
tot2[cTopRight] += val;
}
// printf ("\n");
}
// printf ("\n");
if (t2 || quad) {
t2max = tot2[0];
c = cBottomLeft;
for (int i = 1; i < 4; i++) {
if (tot2[i] > t2max) {
t2max = tot2[i];
c = i;
}
}
// printf("*** %d %d %d %d --
// %d\n",tot2[0],tot2[1],tot2[2],tot2[3],t2max);
if (quad)
*quad = c;
if (t2)
*t2 = t2max;
}
if (t3)
*t3 = tot3;
if (eta2x || eta2y) {
if (eta2x)
*eta2x = 0;
if (eta2y)
*eta2y = 0;
switch (c) {
case cBottomLeft:
if (eta2x && (data[3] + data[4]) != 0)
*eta2x = static_cast<double>(data[4]) / (data[3] + data[4]);
if (eta2y && (data[1] + data[4]) != 0)
*eta2y = static_cast<double>(data[4]) / (data[1] + data[4]);
break;
case cBottomRight:
if (eta2x && (data[2] + data[5]) != 0)
*eta2x = static_cast<double>(data[5]) / (data[4] + data[5]);
if (eta2y && (data[1] + data[4]) != 0)
*eta2y = static_cast<double>(data[4]) / (data[1] + data[4]);
break;
case cTopLeft:
if (eta2x && (data[7] + data[4]) != 0)
*eta2x = static_cast<double>(data[4]) / (data[3] + data[4]);
if (eta2y && (data[7] + data[4]) != 0)
*eta2y = static_cast<double>(data[7]) / (data[7] + data[4]);
break;
case cTopRight:
if (eta2x && t2max != 0)
*eta2x = static_cast<double>(data[5]) / (data[5] + data[4]);
if (eta2y && t2max != 0)
*eta2y = static_cast<double>(data[7]) / (data[7] + data[4]);
break;
default:;
}
}
if (eta3x || eta3y) {
if (eta3x && (data[3] + data[4] + data[5]) != 0)
*eta3x = static_cast<double>(-data[3] + data[3 + 2]) /
(data[3] + data[4] + data[5]);
if (eta3y && (data[1] + data[4] + data[7]) != 0)
*eta3y = static_cast<double>(-data[1] + data[2 * 3 + 1]) /
(data[1] + data[4] + data[7]);
}
return ok;
}
} // namespace aare

80
src/ClusterFile.test.cpp Normal file
View File

@ -0,0 +1,80 @@
#include "aare/ClusterFile.hpp"
#include "test_config.hpp"
#include "aare/defs.hpp"
#include <catch2/catch_test_macros.hpp>
#include <filesystem>
using aare::ClusterFile;
TEST_CASE("Read one frame from a a cluster file", "[.integration]") {
//We know that the frame has 97 clusters
auto fpath = test_data_path() / "clusters" / "single_frame_97_clustrers.clust";
REQUIRE(std::filesystem::exists(fpath));
ClusterFile f(fpath);
auto clusters = f.read_frame();
REQUIRE(clusters.size() == 97);
REQUIRE(clusters.frame_number() == 135);
}
TEST_CASE("Read one frame using ROI", "[.integration]") {
//We know that the frame has 97 clusters
auto fpath = test_data_path() / "clusters" / "single_frame_97_clustrers.clust";
REQUIRE(std::filesystem::exists(fpath));
ClusterFile f(fpath);
aare::ROI roi;
roi.xmin = 0;
roi.xmax = 50;
roi.ymin = 200;
roi.ymax = 249;
f.set_roi(roi);
auto clusters = f.read_frame();
REQUIRE(clusters.size() == 49);
REQUIRE(clusters.frame_number() == 135);
//Check that all clusters are within the ROI
for (size_t i = 0; i < clusters.size(); i++) {
auto c = clusters.at<aare::Cluster3x3>(i);
REQUIRE(c.x >= roi.xmin);
REQUIRE(c.x <= roi.xmax);
REQUIRE(c.y >= roi.ymin);
REQUIRE(c.y <= roi.ymax);
}
}
TEST_CASE("Read clusters from single frame file", "[.integration]") {
auto fpath = test_data_path() / "clusters" / "single_frame_97_clustrers.clust";
REQUIRE(std::filesystem::exists(fpath));
SECTION("Read fewer clusters than available") {
ClusterFile f(fpath);
auto clusters = f.read_clusters(50);
REQUIRE(clusters.size() == 50);
REQUIRE(clusters.frame_number() == 135);
}
SECTION("Read more clusters than available") {
ClusterFile f(fpath);
// 100 is the maximum number of clusters read
auto clusters = f.read_clusters(100);
REQUIRE(clusters.size() == 97);
REQUIRE(clusters.frame_number() == 135);
}
SECTION("Read all clusters") {
ClusterFile f(fpath);
auto clusters = f.read_clusters(97);
REQUIRE(clusters.size() == 97);
REQUIRE(clusters.frame_number() == 135);
}
}