wip

slsReceiverSoftware/src/BinaryDataFile.cpp

@@ -1,7 +1,7 @@
 #include "BinaryDataFile.h"
 #include "sls/logger.h"
 
-BinaryDataFile::BinaryDataFile(int index) : File(index, BINARY) {}
+BinaryDataFile::BinaryDataFile(int index) : File(BINARY), index_(index) {}
 
 BinaryDataFile::~BinaryDataFile() { CloseFile(); }
 
@@ -11,3 +11,36 @@ void BinaryDataFile::CloseFile() {
     }
     fd_ = nullptr;
 }
+
+void BinaryDataFile::CreateDataFile(std::string filePath,
+                                    std::string fileNamePrefix,
+                                    uint64_t fileIndex, bool overWriteEnable,
+                                    bool silentMode, int detIndex,
+                                    int numUnitsPerDetector,
+                                    uint32_t udpPortNumber) {
+    numFramesInFile_ = 0;
+
+    std::ostringstream os;
+    os << filePath << "/" << fileNamePrefix << "_d"
+       << (detIndex * numUnitsPerDetector + index_) << "_f" << subFileIndex_
+       << '_' << fileIndex << ".raw";
+    fileName_ = os.str();
+
+    if (!overWriteEnable) {
+        if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
+            fd_ = nullptr;
+            throw sls::RuntimeError("Could not create/overwrite file " +
+                                    fileName_);
+        }
+    } else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
+        fd_ = nullptr;
+        throw sls::RuntimeError("Could not create file " + fileName_);
+    }
+    // setting to no file buffering
+    setvbuf(fd_, nullptr, _IONBF, 0);
+
+    if (!(silentMode)) {
+        LOG(logINFO) << "[" << udpPortNumber
+                     << "]: Binary File created: " << fileName_;
+    }
+}

slsReceiverSoftware/src/BinaryDataFile.h

@@ -9,7 +9,15 @@ class BinaryDataFile : private virtual slsDetectorDefs, public File {
     ~BinaryDataFile();
 
     void CloseFile() override;
+    void CreateDataFile(std::string filePath, std::string fileNamePrefix,
+                        uint64_t fileIndex, bool overWriteEnable,
+                        bool silentMode, int detIndex, int numUnitsPerDetector,
+                        uint32_t udpPortNumber) override;
 
   private:
+    uint32_t index_;
     FILE *fd_{nullptr};
+    std::string fileName_;
+    uint32_t numFramesInFile_{0};
+    uint32_t subFileIndex_{0}; // to do reset in createfirstdatafile
 };

slsReceiverSoftware/src/BinaryMasterFile.cpp

@@ -2,7 +2,7 @@
 #include "MasterAttributes.h"
 #include "sls/logger.h"
 
-BinaryMasterFile::BinaryMasterFile(int index) : File(index, BINARY) {}
+BinaryMasterFile::BinaryMasterFile() : File(BINARY) {}
 
 BinaryMasterFile::~BinaryMasterFile() { CloseFile(); }
 
@@ -13,11 +13,11 @@ void BinaryMasterFile::CloseFile() {
     fd_ = nullptr;
 }
 
-void BinaryMasterFile::CreateMasterFile(MasterAttributes *attr,
+void BinaryMasterFile::CreateMasterFile(std::string filePath,
-                                        std::string filePath,
                                         std::string fileNamePrefix,
                                         uint64_t fileIndex,
-                                        bool overWriteEnable, bool silentMode) {
+                                        bool overWriteEnable, bool silentMode,
+                                        MasterAttributes *attr) {
     // create file name
     std::ostringstream os;
     os << filePath << "/" << fileNamePrefix << "_master"
@@ -31,14 +31,13 @@ void BinaryMasterFile::CreateMasterFile(MasterAttributes *attr,
     if (!(overWriteEnable)) {
         if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
             fd_ = nullptr;
-            throw sls::RuntimeError(
+            throw sls::RuntimeError("Could not create binary master file " +
-                "Could not create binary master file (no overwrite) " +
+                                    fileName_);
-                fileName_);
         }
     } else if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "w"))) {
         fd_ = nullptr;
         throw sls::RuntimeError(
-            "Could not create binary master file (overwrite) " + fileName_);
+            "Could not create/overwrite binary master file " + fileName_);
     }
 
     attr->WriteMasterBinaryAttributes(fd_);

slsReceiverSoftware/src/BinaryMasterFile.h

@@ -6,13 +6,13 @@
 class BinaryMasterFile : private virtual slsDetectorDefs, public File {
 
   public:
-    BinaryMasterFile(int index);
+    BinaryMasterFile();
     ~BinaryMasterFile();
 
     void CloseFile() override;
-    void CreateMasterFile(MasterAttributes *attr, std::string filePath,
+    void CreateMasterFile(std::string filePath, std::string fileNamePrefix,
-                          std::string fileNamePrefix, uint64_t fileIndex,
+                          uint64_t fileIndex, bool overWriteEnable,
-                          bool overWriteEnable, bool silentMode) override;
+                          bool silentMode, MasterAttributes *attr) override;
 
   private:
     FILE *fd_{nullptr};

slsReceiverSoftware/src/File.cpp

@@ -2,8 +2,7 @@
 
 #include <iostream>
 
-File::File(int index, slsDetectorDefs::fileFormat type)
+File::File(slsDetectorDefs::fileFormat type) : type_(type) {}
-    : index_(index), type_(type) {}
 
 File::~File() {}
 

slsReceiverSoftware/src/File.h

@@ -7,18 +7,32 @@ struct MasterAttributes;
 class File : private virtual slsDetectorDefs {
 
   public:
-    File(int index, slsDetectorDefs::fileFormat type);
+    File(slsDetectorDefs::fileFormat type);
     virtual ~File();
 
     fileFormat GetFileType();
-
     virtual void CloseFile() = 0;
-    virtual void CreateMasterFile(MasterAttributes *attr, std::string filePath,
+
+    virtual void CreateMasterFile(std::string filePath,
                                   std::string fileNamePrefix,
                                   uint64_t fileIndex, bool overWriteEnable,
-                                  bool silentMode) = 0;
+                                  bool silentMode, MasterAttributes *attr) = 0;
+
+    // hdf5 only
+    virtual void
+    CreateFirstDataFile(std::string filePath, std::string fileNamePrefix,
+                        uint64_t fileIndex, bool overWriteEnable,
+                        bool silentMode, int detIndex, int numUnitsPerDetector,
+                        uint32_t udpPortNumber, uint32_t maxFramesPerFile,
+                        uint64_t numImages, uint32_t nPIxelsX,
+                        uint32_t nPIxelsY, uint32_t dynamicRange) = 0;
+
+    virtual void CreateDataFile(std::string filePath,
+                                std::string fileNamePrefix, uint64_t fileIndex,
+                                bool overWriteEnable, bool silentMode,
+                                int detIndex, int numUnitsPerDetector,
+                                uint32_t udpPortNumber) = 0;
 
   protected:
-    int index_;
     slsDetectorDefs::fileFormat type_;
 };

slsReceiverSoftware/src/HDF5DataFile.cpp

@@ -1,11 +1,41 @@
 #include "HDF5DataFile.h"
+#include "receiver_defs.h"
 #include "sls/logger.h"
 
-HDF5DataFile::HDF5DataFile(int index) : File(index, HDF5) {}
+#include <iomanip>
+
+HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
+    : File(HDF5), index_(index), hdf5Lib_(hdf5Lib) {
+
+    parameterNames_ = std::vector<std::string>{
+        "frame number",
+        "exp length or sub exposure time",
+        "packets caught",
+        "bunch id",
+        "timestamp",
+        "mod id",
+        "row",
+        "column",
+        "reserved",
+        "debug",
+        "round robin number",
+        "detector type",
+        "detector header version",
+        "packets caught bit mask",
+    };
+    StrType strdatatype(PredType::C_S1, sizeof(bitset_storage));
+    parameterDataTypes_ = std::vector<DataType>{
+        PredType::STD_U64LE, PredType::STD_U32LE, PredType::STD_U32LE,
+        PredType::STD_U64LE, PredType::STD_U64LE, PredType::STD_U16LE,
+        PredType::STD_U16LE, PredType::STD_U16LE, PredType::STD_U16LE,
+        PredType::STD_U32LE, PredType::STD_U16LE, PredType::STD_U8LE,
+        PredType::STD_U8LE,  strdatatype};
+}
 
 HDF5DataFile::~HDF5DataFile() { CloseFile(); }
 
 void HDF5DataFile::CloseFile() {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
     try {
         Exception::dontPrint(); // to handle errors
         if (fd_) {
@@ -18,4 +48,139 @@ void HDF5DataFile::CloseFile() {
                       << index_;
         error.printErrorStack();
     }
+    if (dataSpace_) {
+        delete dataSpace_;
+        dataSpace_ = nullptr;
+    }
+    if (dataSet_) {
+        delete dataSet_;
+        dataSet_ = nullptr;
+    }
+    if (dataSpacePara_) {
+        delete dataSpacePara_;
+        dataSpacePara_ = nullptr;
+    }
+    for (auto it : dataSetPara_)
+        delete it;
+    dataSetPara_.clear();
+}
+
+void HDF5DataFile::CreateFirstDataFile(
+    std::string filePath, std::string fileNamePrefix, uint64_t fileIndex,
+    bool overWriteEnable, bool silentMode, int detIndex,
+    int numUnitsPerDetector, uint32_t udpPortNumber, uint32_t maxFramesPerFile,
+    uint64_t numImages, uint32_t nPIxelsX, uint32_t nPIxelsY,
+    uint32_t dynamicRange) {
+
+    CreateDataFile(filePath, fileNamePrefix, fileIndex, overWriteEnable,
+                   silentMode, detIndex, numUnitsPerDetector, udpPortNumber);
+
+    maxFramesPerFile_ = maxFramesPerFile;
+    numImages_ = numImages;
+    nPixelsX_ = nPIxelsX;
+    nPixelsY_ = nPIxelsY;
+    dynamicRange_ = dynamicRange;
+}
+
+void HDF5DataFile::CreateDataFile(std::string filePath,
+                                  std::string fileNamePrefix,
+                                  uint64_t fileIndex, bool overWriteEnable,
+                                  bool silentMode, int detIndex,
+                                  int numUnitsPerDetector,
+                                  uint32_t udpPortNumber) {
+
+    std::ostringstream os;
+    os << filePath << "/" << fileNamePrefix << "_d"
+       << (detIndex * numUnitsPerDetector + index_) << "_f" << subFileIndex_
+       << '_' << fileIndex << ".h5";
+    fileName_ = os.str();
+
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+
+    uint64_t framestosave =
+        ((maxFramesPerFile_ == 0) ? numImages_ : // infinite images
+             (((extNumImages_ - subFileIndex_) > maxFramesPerFile_)
+                  ? // save up to maximum at a time
+                  maxFramesPerFile_
+                  : (extNumImages_ - subFileIndex_)));
+
+    uint64_t nDimx = framestosave;
+    uint32_t nDimy = nPixelsY_;
+    uint32_t nDimz = ((dynamicRange_ == 4) ? (nPixelsX_ / 2) : nPixelsX_);
+
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        // file
+        FileAccPropList fapl;
+        fapl.setFcloseDegree(H5F_CLOSE_STRONG);
+        fd_ = nullptr;
+        if (!overWriteEnable)
+            fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
+                             FileCreatPropList::DEFAULT, fapl);
+        else
+            fd_ = new H5File(fileName_.c_str(), H5F_ACC_TRUNC,
+                             FileCreatPropList::DEFAULT, fapl);
+
+        // attributes - version
+        double dValue = HDF5_WRITER_VERSION;
+        DataSpace dataspace_attr = DataSpace(H5S_SCALAR);
+        Attribute attribute = fd_->createAttribute(
+            "version", PredType::NATIVE_DOUBLE, dataspace_attr);
+        attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+        // dataspace
+        hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
+        hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
+        dataSpace_ = nullptr;
+        dataSpace_ = new DataSpace(3, srcdims, srcdimsmax);
+
+        // dataset name
+        std::ostringstream osfn;
+        osfn << "/data";
+        if (numImages_ > 1)
+            osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex_;
+        std::string dsetname = osfn.str();
+
+        // dataset
+        // fill value
+        DSetCreatPropList plist;
+        int fill_value = -1;
+        plist.setFillValue(dataType, &fill_value);
+        // always create chunked dataset as unlimited is only
+        // supported with chunked layout
+        hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
+        plist.setChunk(3, chunk_dims);
+        dataSet_ = nullptr;
+        dataSet_ = new DataSet(
+            fd_->createDataSet(dsetname.c_str(), dataType, *dataSpace_, plist));
+
+        // create parameter datasets
+        hsize_t dims[1] = {nDimx};
+        hsize_t dimsmax[1] = {H5S_UNLIMITED};
+        dataSpacePara_ = nullptr;
+        dataSpacePara_ = new DataSpace(1, dims, dimsmax);
+
+        // always create chunked dataset as unlimited is only
+        // supported with chunked layout
+        DSetCreatPropList paralist;
+        hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
+        paralist.setChunk(1, chunkpara_dims);
+
+        for (unsigned int i = 0; i < parameterNames_.size(); ++i) {
+            DataSet *ds = new DataSet(fd_->createDataSet(
+                parameterNames_[i].c_str(), parameterDataTypes_[i],
+                *dataSpacePara_, paralist));
+            dataSetPara_.push_back(ds);
+        }
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        CloseFile();
+        throw sls::RuntimeError("Could not create HDF5 handles in object " +
+                                index_);
+    }
+    if (!(silentMode)) {
+        LOG(logINFO) << "[" << udpPortNumber
+                     << "]: HDF5 File created: " << fileName_;
+    }
 }

slsReceiverSoftware/src/HDF5DataFile.h

@@ -7,14 +7,48 @@
 using namespace H5;
 #endif
 
+#include <mutex>
+
 class HDF5DataFile : private virtual slsDetectorDefs, public File {
 
   public:
-    HDF5DataFile(int index);
+    HDF5DataFile(int index, std::mutex *hdf5Lib);
     ~HDF5DataFile();
 
     void CloseFile() override;
 
+    void CreateFirstDataFile(std::string filePath, std::string fileNamePrefix,
+                             uint64_t fileIndex, bool overWriteEnable,
+                             bool silentMode, int detIndex,
+                             int numUnitsPerDetector, uint32_t udpPortNumber,
+                             uint32_t maxFramesPerFile, uint64_t numImages,
+                             uint32_t nPIxelsX, uint32_t nPIxelsY,
+                             uint32_t dynamicRange) override;
+
+    void CreateDataFile(std::string filePath, std::string fileNamePrefix,
+                        uint64_t fileIndex, bool overWriteEnable,
+                        bool silentMode, int detIndex, int numUnitsPerDetector,
+                        uint32_t udpPortNumber) override;
+
   private:
+    int index_;
+    std::mutex *hdf5Lib_;
     H5File *fd_{nullptr};
+    std::string fileName_;
+    DataSpace *dataSpace_{nullptr};
+    DataSet *dataSet_{nullptr};
+    DataType dataType{PredType::STD_U16LE};
+
+    DataSpace *dataSpacePara_{nullptr};
+    std::vector<DataSet *> dataSetPara_{nullptr};
+    std::vector<std::string> parameterNames_;
+    std::vector<DataType> parameterDataTypes_;
+
+    uint32_t subFileIndex_{0}; // to do reset in createfirstdatafile
+    uint32_t maxFramesPerFile_{0};
+    uint64_t numImages_{0};
+    uint64_t extNumImages_{0}; // TODO needs to be defined in startofacquisition
+    uint32_t nPixelsX_{0};
+    uint32_t nPixelsY_{0};
+    uint32_t dynamicRange_{0};
 };

slsReceiverSoftware/src/HDF5MasterFile.cpp

@@ -2,12 +2,13 @@
 #include "MasterAttributes.h"
 #include "sls/logger.h"
 
-HDF5MasterFile::HDF5MasterFile(int index, std::mutex *hdf5Lib)
+HDF5MasterFile::HDF5MasterFile(std::mutex *hdf5Lib)
-    : File(index, HDF5), hdf5Lib_(hdf5Lib) {}
+    : File(HDF5), hdf5Lib_(hdf5Lib) {}
 
 HDF5MasterFile::~HDF5MasterFile() { CloseFile(); }
 
 void HDF5MasterFile::CloseFile() {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
     try {
         Exception::dontPrint(); // to handle errors
         if (fd_) {
@@ -16,17 +17,15 @@ void HDF5MasterFile::CloseFile() {
             fd_ = nullptr;
         }
     } catch (const Exception &error) {
-        LOG(logERROR) << "Could not close data HDF5 handles of index "
+        LOG(logERROR) << "Could not close master HDF5 handles";
-                      << index_;
         error.printErrorStack();
     }
 }
 
-void HDF5MasterFile::CreateMasterFile(MasterAttributes *attr,
+void HDF5MasterFile::CreateMasterFile(std::string filePath,
-                                      std::string filePath,
                                       std::string fileNamePrefix,
                                       uint64_t fileIndex, bool overWriteEnable,
-                                      bool silentMode) {
+                                      bool silentMode, MasterAttributes *attr) {
 
     std::ostringstream os;
     os << filePath << "/" << fileNamePrefix << "_master"
@@ -66,6 +65,7 @@ void HDF5MasterFile::CreateMasterFile(MasterAttributes *attr,
     } catch (const Exception &error) {
         error.printErrorStack();
         CloseFile();
-        throw sls::RuntimeError("Could not create master HDF5 handles");
+        throw sls::RuntimeError(
+            "Could not create/overwrite master HDF5 handles");
     }
 }

slsReceiverSoftware/src/HDF5MasterFile.h

@@ -12,13 +12,13 @@ using namespace H5;
 class HDF5MasterFile : private virtual slsDetectorDefs, public File {
 
   public:
-    HDF5MasterFile(int index, std::mutex *hdf5Lib);
+    HDF5MasterFile(std::mutex *hdf5Lib);
     ~HDF5MasterFile();
 
     void CloseFile() override;
-    void CreateMasterFile(MasterAttributes *attr, std::string filePath,
+    void CreateMasterFile(std::string filePath, std::string fileNamePrefix,
-                          std::string fileNamePrefix, uint64_t fileIndex,
+                          uint64_t fileIndex, bool overWriteEnable,
-                          bool overWriteEnable, bool silentMode) override;
+                          bool silentMode, MasterAttributes *attr) override;
 
   private:
     std::mutex *hdf5Lib_;

slsReceiverSoftware/src/HDF5VirtualFile.cpp

@@ -1,15 +1,28 @@
 #include "HDF5VirtualFile.h"
 #include "sls/logger.h"
 
-HDF5VirtualFile::HDF5VirtualFile(int index) : File(index, HDF5), fd_(0) {}
+HDF5VirtualFile::HDF5VirtualFile(std::mutex *hdf5Lib)
+    : File(HDF5), hdf5Lib_(hdf5Lib) {}
 
 HDF5VirtualFile::~HDF5VirtualFile() { CloseFile(); }
 
 void HDF5VirtualFile::CloseFile() {
-    if (fd_ != 0) {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+    /*if (fd_ != 0) {
         if (H5Fclose(fd_) < 0) {
             LOG(logERROR) << "Could not close virtual HDF5 handles";
         }
         fd_ = 0;
+    }*/
+    try {
+        Exception::dontPrint(); // to handle errors
+        if (fd_) {
+            fd_->close();
+            delete fd_;
+            fd_ = nullptr;
+        }
+    } catch (const Exception &error) {
+        LOG(logERROR) << "Could not close virtual HDF5 handles of index";
+        error.printErrorStack();
     }
 }

slsReceiverSoftware/src/HDF5VirtualFile.h

@@ -7,14 +7,18 @@
 using namespace H5;
 #endif
 
+#include <mutex>
+
 class HDF5VirtualFile : private virtual slsDetectorDefs, public File {
 
   public:
-    HDF5VirtualFile(int index);
+    HDF5VirtualFile(std::mutex *hdf5Lib);
     ~HDF5VirtualFile();
 
     void CloseFile() override;
 
   private:
-    hid_t fd_;
+    std::mutex *hdf5Lib_;
+    // hid_t fd_;
+    H5File *fd_{nullptr};
 };

slsReceiverSoftware/src/MasterAttributes.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "receiver_defs.h"
 #include "sls/ToString.h"
 #include "sls/logger.h"
 #include "sls/sls_detector_defs.h"
@@ -14,10 +15,6 @@ using namespace H5;
 #include <chrono>
 using ns = std::chrono::nanoseconds;
 
-// versions
-#define HDF5_WRITER_VERSION   (6.2) // 1 decimal places
-#define BINARY_WRITER_VERSION (6.2) // 1 decimal places
-
 struct MasterAttributes {
     slsDetectorDefs::detectorType detType{slsDetectorDefs::GENERIC};
     slsDetectorDefs::timingMode timingMode{slsDetectorDefs::AUTO_TIMING};
@@ -36,7 +33,7 @@ struct MasterAttributes {
     uint32_t dynamicRange{0};
     uint32_t tenGiga{0};
     int thresholdEnergyeV{0};
-  std::array<int, 3> thresholdAllEnergyeV={{0, 0, 0}};
+    std::array<int, 3> thresholdAllEnergyeV = {{0, 0, 0}};
     ns subExptime{0};
     ns subPeriod{0};
     uint32_t quad{0};

slsReceiverSoftware/src/receiver_defs.h

@@ -13,6 +13,11 @@
 #define MAX_SOCKET_INPUT_PACKET_QUEUE (250000)
 
 // files
+
+// versions
+#define HDF5_WRITER_VERSION   (6.2) // 1 decimal places
+#define BINARY_WRITER_VERSION (6.2) // 1 decimal places
+
 #define MAX_FRAMES_PER_FILE           20000
 #define SHORT_MAX_FRAMES_PER_FILE     100000
 #define MOENCH_MAX_FRAMES_PER_FILE    100000