frames in file added for master file in receiver, binary done, hdf5 not done

slsReceiverSoftware/src/BinaryMasterFile.cpp

@@ -42,3 +42,17 @@ void BinaryMasterFile::CreateMasterFile(const std::string filePath,
     attr->WriteMasterBinaryAttributes(fd_);
     CloseFile();
 }
+
+void BinaryMasterFile::UpdateMasterFile(MasterAttributes *attr,
+                                        bool silentMode) {
+    if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "a"))) {
+        fd_ = nullptr;
+        throw sls::RuntimeError("Could not append binary master file " +
+                                fileName_);
+    }
+    attr->WriteFinalBinaryAttributes(fd_);
+    CloseFile();
+    if (!silentMode) {
+        LOG(logINFO) << "Updated Master File";
+    }
+}

slsReceiverSoftware/src/BinaryMasterFile.h

@@ -15,6 +15,7 @@ class BinaryMasterFile : private virtual slsDetectorDefs, public File {
                           const uint64_t fileIndex, const bool overWriteEnable,
                           const bool silentMode,
                           MasterAttributes *attr) override;
+    void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
 
   private:
     FILE *fd_{nullptr};

slsReceiverSoftware/src/DataProcessor.cpp

@@ -17,6 +17,7 @@
 #include "HDF5VirtualFile.h"
 #endif
 #include "DataStreamer.h"
+#include "sls/container_utils.h"
 #include "sls/sls_detector_exceptions.h"
 
 #include <cerrno>
@@ -56,6 +57,10 @@ bool DataProcessor::GetStartedFlag() { return startedFlag_; }
 
 uint64_t DataProcessor::GetNumFramesCaught() { return numFramesCaught_; }
 
+uint64_t DataProcessor::GetNumCompleteFramesCaught() {
+    return numCompleteFramesCaught_;
+}
+
 uint64_t DataProcessor::GetCurrentFrameIndex() { return currentFrameIndex_; }
 
 uint64_t DataProcessor::GetProcessedIndex() {
@@ -68,6 +73,7 @@ void DataProcessor::ResetParametersforNewAcquisition() {
     StopRunning();
     startedFlag_ = false;
     numFramesCaught_ = 0;
+    numCompleteFramesCaught_ = 0;
     firstIndex_ = 0;
     currentFrameIndex_ = 0;
     firstStreamerFrame_ = true;
@@ -248,6 +254,41 @@ void DataProcessor::LinkDataInMasterFile(const bool silentMode) {
 }
 #endif
 
+void DataProcessor::UpdateMasterFile(bool silentMode) {
+    if (masterFile_) {
+        // final attributes
+        std::unique_ptr<MasterAttributes> masterAttributes;
+        switch (detectorType_) {
+        case GOTTHARD:
+            masterAttributes = sls::make_unique<GotthardMasterAttributes>();
+            break;
+        case JUNGFRAU:
+            masterAttributes = sls::make_unique<JungfrauMasterAttributes>();
+            break;
+        case EIGER:
+            masterAttributes = sls::make_unique<EigerMasterAttributes>();
+            break;
+        case MYTHEN3:
+            masterAttributes = sls::make_unique<Mythen3MasterAttributes>();
+            break;
+        case GOTTHARD2:
+            masterAttributes = sls::make_unique<Gotthard2MasterAttributes>();
+            break;
+        case MOENCH:
+            masterAttributes = sls::make_unique<MoenchMasterAttributes>();
+            break;
+        case CHIPTESTBOARD:
+            masterAttributes = sls::make_unique<CtbMasterAttributes>();
+            break;
+        default:
+            throw sls::RuntimeError(
+                "Unknown detector type to set up master file attributes");
+        }
+        masterAttributes->framesInFile = numFramesCaught_;
+        masterFile_->UpdateMasterFile(masterAttributes.get(), silentMode);
+    }
+}
+
 void DataProcessor::ThreadExecution() {
     char *buffer = nullptr;
     fifo_->PopAddress(buffer);
@@ -306,9 +347,10 @@ uint64_t DataProcessor::ProcessAnImage(char *buf) {
     sls_detector_header header = rheader->detHeader;
     uint64_t fnum = header.frameNumber;
     currentFrameIndex_ = fnum;
+    numFramesCaught_++;
     uint32_t nump = header.packetNumber;
     if (nump == generalData_->packetsPerFrame) {
-        numFramesCaught_++;
+        numCompleteFramesCaught_++;
     }
 
     LOG(logDEBUG1) << "DataProcessing " << index << ": fnum:" << fnum;

slsReceiverSoftware/src/DataProcessor.h

@@ -37,6 +37,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
 
     bool GetStartedFlag();
     uint64_t GetNumFramesCaught();
+    uint64_t GetNumCompleteFramesCaught();
     /** (-1 if no frames have been caught */
     uint64_t GetCurrentFrameIndex();
     /** (-1 if no frames have been caught) */
@@ -76,6 +77,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
                            const int numModY);
     void LinkDataInMasterFile(const bool silentMode);
 #endif
+    void UpdateMasterFile(bool silentMode);
     /**
      * Call back for raw data
      * args to raw data ready callback are
@@ -174,9 +176,12 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
     std::atomic<uint64_t> firstIndex_{0};
 
     // for statistics
-    /** Number of complete frames caught */
+    /** Number of frames caught */
     uint64_t numFramesCaught_{0};
 
+    /** Number of complete frames caught */
+    uint64_t numCompleteFramesCaught_{0};
+
     /** Frame Number of latest processed frame number */
     std::atomic<uint64_t> currentFrameIndex_{0};
 

slsReceiverSoftware/src/File.h

@@ -118,6 +118,11 @@ class File : private virtual slsDetectorDefs {
                          "should be overloaded by a derived class";
     };
 
+    virtual void UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
+        LOG(logERROR) << "This is a generic function UpdateMasterFile that "
+                         "should be overloaded by a derived class";
+    };
+
   protected:
     slsDetectorDefs::fileFormat format_;
 };

slsReceiverSoftware/src/HDF5MasterFile.cpp

@@ -117,6 +117,7 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
         Group group5(group3.createGroup("detector"));
         Group group6(group1.createGroup("sample"));
 
+        groupId_ = group5.getId();
         attr->WriteMasterHDF5Attributes(fd_, &group5);
         fd_->close();
 
@@ -130,3 +131,31 @@ void HDF5MasterFile::CreateMasterFile(const std::string filePath,
         LOG(logINFO) << "Master File: " << fileName_;
     }
 }
+
+void HDF5MasterFile::UpdateMasterFile(MasterAttributes *attr, bool silentMode) {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        FileAccPropList flist;
+        flist.setFcloseDegree(H5F_CLOSE_STRONG);
+        fd_ = new H5File(fileName_.c_str(), H5F_ACC_RDWR,
+                         FileCreatPropList::DEFAULT, flist);
+
+        Group group(groupId_);
+
+        // cannot do this TODO
+        attr->WriteFinalHDF5Attributes(fd_, &group);
+        fd_->close();
+
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        CloseFile();
+        throw sls::RuntimeError(
+            "Could not create/overwrite master HDF5 handles");
+    }
+    if (!silentMode) {
+        LOG(logINFO) << "Updated Master File";
+    }
+}

slsReceiverSoftware/src/HDF5MasterFile.h

@@ -19,9 +19,11 @@ class HDF5MasterFile : private virtual slsDetectorDefs, public File {
                           const uint64_t fileIndex, const bool overWriteEnable,
                           const bool silentMode,
                           MasterAttributes *attr) override;
+    void UpdateMasterFile(MasterAttributes *attr, bool silentMode) override;
 
   private:
     std::mutex *hdf5Lib_;
     H5File *fd_{nullptr};
     std::string fileName_;
+    hid_t groupId_{0};
 };

slsReceiverSoftware/src/Implementation.cpp

@@ -429,7 +429,7 @@ uint64_t Implementation::getFramesCaught() const {
 
     for (const auto &it : dataProcessor) {
         flagsum += it->GetStartedFlag();
-        min = std::min(min, it->GetNumFramesCaught());
+        min = std::min(min, it->GetNumCompleteFramesCaught());
     }
     // no data processed
     if (flagsum != dataProcessor.size())
@@ -571,6 +571,13 @@ void Implementation::stopReceiver() {
         }
     }
 #endif
+    if (fileWriteEnable && masterFileWriteEnable && modulePos == 0) {
+        try {
+            dataProcessor[0]->UpdateMasterFile(silentMode);
+        } catch (...) {
+            ; // ignore it and just print it
+        }
+    }
 
     // wait for the processes (dataStreamer) to be done
     running = true;
@@ -589,7 +596,7 @@ void Implementation::stopReceiver() {
         std::vector<uint64_t> mp = getNumMissingPackets();
         uint64_t tot = 0;
         for (int i = 0; i < numThreads; i++) {
-            int nf = dataProcessor[i]->GetNumFramesCaught();
+            int nf = dataProcessor[i]->GetNumCompleteFramesCaught();
             tot += nf;
             std::string mpMessage = std::to_string((int64_t)mp[i]);
             if ((int64_t)mp[i] < 0) {

slsReceiverSoftware/src/MasterAttributes.h

@@ -16,6 +16,7 @@ using namespace H5;
 using ns = std::chrono::nanoseconds;
 
 struct MasterAttributes {
+    // (before acquisition)
     slsDetectorDefs::detectorType detType{slsDetectorDefs::GENERIC};
     slsDetectorDefs::timingMode timingMode{slsDetectorDefs::AUTO_TIMING};
     uint32_t imageSize{0};
@@ -57,6 +58,9 @@ struct MasterAttributes {
     uint32_t gates;
     std::map<std::string, std::string> additionalJsonHeader;
 
+    // Final Attributes (after acquisition)
+    uint64_t framesInFile{0};
+
     MasterAttributes(){};
     virtual ~MasterAttributes(){};
 
@@ -87,30 +91,42 @@ struct MasterAttributes {
     };
 
     void WriteBinaryAttributes(FILE *fd, std::string message) {
+        if (fwrite((void *)message.c_str(), 1, message.length(), fd) !=
+            message.length()) {
+            throw sls::RuntimeError(
+                "Master binary file incorrect number of bytes written to file");
+        }
+    };
+
+    void WriteFinalBinaryAttributes(FILE *fd) {
         // adding few common parameters to the end
+        std::ostringstream oss;
+
         if (!additionalJsonHeader.empty()) {
-            std::ostringstream oss;
             oss << "Additional Json Header     : "
                 << sls::ToString(additionalJsonHeader) << '\n';
-            message += oss.str();
         }
+        oss << "Frames in File             : " << framesInFile << '\n';
 
         // adding sls_receiver header format
-        message += std::string("\n#Frame Header\n"
+        oss << '\n'
-                               "Frame Number               : 8 bytes\n"
+            << "#Frame Header" << '\n'
-                               "SubFrame Number/ExpLength  : 4 bytes\n"
+            << "Frame Number               : 8 bytes" << '\n'
-                               "Packet Number              : 4 bytes\n"
+            << "SubFrame Number/ExpLength  : 4 bytes" << '\n'
-                               "Bunch ID                   : 8 bytes\n"
+            << "Packet Number              : 4 bytes" << '\n'
-                               "Timestamp                  : 8 bytes\n"
+            << "Bunch ID                   : 8 bytes" << '\n'
-                               "Module Id                  : 2 bytes\n"
+            << "Timestamp                  : 8 bytes" << '\n'
-                               "Row                        : 2 bytes\n"
+            << "Module Id                  : 2 bytes" << '\n'
-                               "Column                     : 2 bytes\n"
+            << "Row                        : 2 bytes" << '\n'
-                               "Reserved                   : 2 bytes\n"
+            << "Column                     : 2 bytes" << '\n'
-                               "Debug                      : 4 bytes\n"
+            << "Reserved                   : 2 bytes" << '\n'
-                               "Round Robin Number         : 2 bytes\n"
+            << "Debug                      : 4 bytes" << '\n'
-                               "Detector Type              : 1 byte\n"
+            << "Round Robin Number         : 2 bytes" << '\n'
-                               "Header Version             : 1 byte\n"
+            << "Detector Type              : 1 byte" << '\n'
-                               "Packets Caught Mask        : 64 bytes\n");
+            << "Header Version             : 1 byte" << '\n'
+            << "Packets Caught Mask        : 64 bytes" << '\n';
+
+        std::string message = oss.str();
 
         // writing to file
         if (fwrite((void *)message.c_str(), 1, message.length(), fd) !=
@@ -233,6 +249,18 @@ struct MasterAttributes {
                 "Total Frames", PredType::STD_U64LE, dataspace);
             dataset.write(&totalFrames, PredType::STD_U64LE);
         }
+    };
+
+    void WriteFinalHDF5Attributes(H5File *fd, Group *group) {
+        char c[1024];
+        memset(c, 0, sizeof(c));
+        // Total Frames in file
+        {
+            DataSpace dataspace = DataSpace(H5S_SCALAR);
+            DataSet dataset = group->createDataSet(
+                "Frames in File", PredType::STD_U64LE, dataspace);
+            dataset.write(&framesInFile, PredType::STD_U64LE);
+        }
         // additional json header
         if (!additionalJsonHeader.empty()) {
             std::string json = sls::ToString(additionalJsonHeader);