wip

slsReceiverSoftware/src/BinaryDataFile.cpp

@@ -12,21 +12,38 @@ void BinaryDataFile::CloseFile() {
     fd_ = nullptr;
 }
 
-void BinaryDataFile::CreateDataFile(std::string filePath,
+void BinaryDataFile::CreateFirstBinaryDataFile(
-                                    std::string fileNamePrefix,
+    std::string filePath, std::string fileNamePrefix, uint64_t fileIndex,
-                                    uint64_t fileIndex, bool overWriteEnable,
+    bool overWriteEnable, bool silentMode, int detIndex,
-                                    bool silentMode, int detIndex,
+    int numUnitsPerDetector, uint32_t udpPortNumber,
-                                    int numUnitsPerDetector,
+    uint32_t maxFramesPerFile) {
-                                    uint32_t udpPortNumber) {
+
+    subFileIndex_ = 0;
+    numFramesInFile_ = 0;
+
+    filePath_ = filePath;
+    fileNamePrefix_ = fileNamePrefix;
+    fileIndex_ = fileIndex;
+    overWriteEnable_ = overWriteEnable;
+    silentMode_ = silentMode;
+    detIndex_ = detIndex;
+    numUnitsPerDetector_ = numUnitsPerDetector;
+    udpPortNumber_ = udpPortNumber;
+    maxFramesPerFile_ = maxFramesPerFile;
+
+    CreateFile();
+}
+
+void BinaryDataFile::CreateFile() {
     numFramesInFile_ = 0;
 
     std::ostringstream os;
-    os << filePath << "/" << fileNamePrefix << "_d"
+    os << filePath_ << "/" << fileNamePrefix_ << "_d"
-       << (detIndex * numUnitsPerDetector + index_) << "_f" << subFileIndex_
+       << (detIndex_ * numUnitsPerDetector_ + index_) << "_f" << subFileIndex_
-       << '_' << fileIndex << ".raw";
+       << '_' << fileIndex_ << ".raw";
     fileName_ = os.str();
 
-    if (!overWriteEnable) {
+    if (!overWriteEnable_) {
         if (nullptr == (fd_ = fopen((const char *)fileName_.c_str(), "wx"))) {
             fd_ = nullptr;
             throw sls::RuntimeError("Could not create/overwrite file " +
@@ -39,8 +56,54 @@ void BinaryDataFile::CreateDataFile(std::string filePath,
     // setting to no file buffering
     setvbuf(fd_, nullptr, _IONBF, 0);
 
-    if (!(silentMode)) {
+    if (!silentMode_) {
-        LOG(logINFO) << "[" << udpPortNumber
+        LOG(logINFO) << "[" << udpPortNumber_
                      << "]: Binary File created: " << fileName_;
     }
 }
+
+void BinaryDataFile::WriteToFile(char *buffer, int buffersize,
+                                 uint64_t currentFrameNumber,
+                                 uint32_t numPacketsCaught) {
+    // check if maxframesperfile = 0 for infinite
+    if (maxFramesPerFile_ && (numFramesInFile_ >= maxFramesPerFile_)) {
+        CloseFile();
+        ++subFileIndex_;
+        CreateFile();
+    }
+    numFramesInFile_++;
+
+    // write to file
+    int ret = 0;
+
+    // contiguous bitset
+    if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
+        ret = fwrite(buffer, 1, buffersize, fd_);
+    }
+
+    // not contiguous bitset
+    else {
+        // write detector header
+        ret = fwrite(buffer, 1, sizeof(sls_detector_header), fd_);
+
+        // get contiguous representation of bit mask
+        bitset_storage storage;
+        memset(storage, 0, sizeof(bitset_storage));
+        sls_bitset bits = *(sls_bitset *)(buffer + sizeof(sls_detector_header));
+        for (int i = 0; i < MAX_NUM_PACKETS; ++i)
+            storage[i >> 3] |= (bits[i] << (i & 7));
+        // write bitmask
+        ret += fwrite((char *)storage, 1, sizeof(bitset_storage), fd_);
+
+        // write data
+        ret += fwrite(buffer + sizeof(sls_detector_header), 1,
+                      buffersize - sizeof(sls_receiver_header), fd_);
+    }
+
+    // if write error
+    if (ret != buffersize) {
+        throw sls::RuntimeError(std::to_string(index_) +
+                                " : Write to file failed for image number " +
+                                std::to_string(currentFrameNumber));
+    }
+}

slsReceiverSoftware/src/BinaryDataFile.h

@@ -9,15 +9,33 @@ class BinaryDataFile : private virtual slsDetectorDefs, public File {
     ~BinaryDataFile();
 
     void CloseFile() override;
-    void CreateDataFile(std::string filePath, std::string fileNamePrefix,
+    void CreateFirstBinaryDataFile(std::string filePath,
+                                   std::string fileNamePrefix,
                                    uint64_t fileIndex, bool overWriteEnable,
-                        bool silentMode, int detIndex, int numUnitsPerDetector,
+                                   bool silentMode, int detIndex,
-                        uint32_t udpPortNumber) override;
+                                   int numUnitsPerDetector,
+                                   uint32_t udpPortNumber,
+                                   uint32_t maxFramesPerFile) override;
+
+    void WriteToFile(char *buffer, int buffersize, uint64_t currentFrameNumber,
+                     uint32_t numPacketsCaught) override;
 
   private:
+    void CreateFile();
+
     uint32_t index_;
     FILE *fd_{nullptr};
     std::string fileName_;
     uint32_t numFramesInFile_{0};
-    uint32_t subFileIndex_{0}; // to do reset in createfirstdatafile
+    uint32_t subFileIndex_{0};
+
+    std::string filePath_;
+    std::string fileNamePrefix_;
+    uint64_t fileIndex_{0};
+    bool overWriteEnable_{false};
+    bool silentMode_{false};
+    int detIndex_{0};
+    int numUnitsPerDetector_{0};
+    uint32_t udpPortNumber_{0};
+    uint32_t maxFramesPerFile_{0};
 };

slsReceiverSoftware/src/File.h

@@ -11,6 +11,7 @@ class File : private virtual slsDetectorDefs {
     virtual ~File();
 
     fileFormat GetFileType();
+    virtual uint32_t GetFilesInAcquisition() = 0;
     virtual void CloseFile() = 0;
 
     virtual void CreateMasterFile(std::string filePath,
@@ -18,20 +19,23 @@ class File : private virtual slsDetectorDefs {
                                   uint64_t fileIndex, bool overWriteEnable,
                                   bool silentMode, MasterAttributes *attr) = 0;
 
-    // hdf5 only
     virtual void
-    CreateFirstDataFile(std::string filePath, std::string fileNamePrefix,
+    CreateFirstBinaryDataFile(std::string filePath, std::string fileNamePrefix,
                               uint64_t fileIndex, bool overWriteEnable,
-                        bool silentMode, int detIndex, int numUnitsPerDetector,
+                              bool silentMode, int detIndex,
-                        uint32_t udpPortNumber, uint32_t maxFramesPerFile,
+                              int numUnitsPerDetector, uint32_t udpPortNumber,
-                        uint64_t numImages, uint32_t nPIxelsX,
+                              uint32_t maxFramesPerFile) = 0;
+
+    virtual void CreateFirstHDF5DataFile(
+        std::string filePath, std::string fileNamePrefix, uint64_t fileIndex,
+        bool overWriteEnable, bool silentMode, int detIndex,
+        int numUnitsPerDetector, uint32_t udpPortNumber,
+        uint32_t maxFramesPerFile, uint64_t numImages, uint32_t nPIxelsX,
         uint32_t nPIxelsY, uint32_t dynamicRange) = 0;
 
-    virtual void CreateDataFile(std::string filePath,
+    virtual void WriteToFile(char *buffer, int buffersize,
-                                std::string fileNamePrefix, uint64_t fileIndex,
+                             uint64_t currentFrameNumber,
-                                bool overWriteEnable, bool silentMode,
+                             uint32_t numPacketsCaught) = 0;
-                                int detIndex, int numUnitsPerDetector,
-                                uint32_t udpPortNumber) = 0;
 
   protected:
     slsDetectorDefs::fileFormat type_;

slsReceiverSoftware/src/HDF5DataFile.cpp

@@ -34,6 +34,10 @@ HDF5DataFile::HDF5DataFile(int index, std::mutex *hdf5Lib)
 
 HDF5DataFile::~HDF5DataFile() { CloseFile(); }
 
+uint32_t HDF5DataFile::GetFilesInAcquisition() {
+    return numFilesInAcquisition_;
+}
+
 void HDF5DataFile::CloseFile() {
     std::lock_guard<std::mutex> lock(*hdf5Lib_);
     try {
@@ -65,34 +69,56 @@ void HDF5DataFile::CloseFile() {
     dataSetPara_.clear();
 }
 
-void HDF5DataFile::CreateFirstDataFile(
+void HDF5DataFile::CreateFirstHDF5DataFile(
     std::string filePath, std::string fileNamePrefix, uint64_t fileIndex,
     bool overWriteEnable, bool silentMode, int detIndex,
     int numUnitsPerDetector, uint32_t udpPortNumber, uint32_t maxFramesPerFile,
     uint64_t numImages, uint32_t nPIxelsX, uint32_t nPIxelsY,
     uint32_t dynamicRange) {
 
-    CreateDataFile(filePath, fileNamePrefix, fileIndex, overWriteEnable,
+    subFileIndex_ = 0;
-                   silentMode, detIndex, numUnitsPerDetector, udpPortNumber);
+    numFramesInFile_ = 0;
+    extNumImages_ = numImages;
+    numFilesInAcquisition_ = 0;
 
     maxFramesPerFile_ = maxFramesPerFile;
     numImages_ = numImages;
     nPixelsX_ = nPIxelsX;
     nPixelsY_ = nPIxelsY;
     dynamicRange_ = dynamicRange;
+
+    filePath_ = filePath;
+    fileNamePrefix_ = fileNamePrefix;
+    fileIndex_ = fileIndex;
+    overWriteEnable_ = overWriteEnable;
+    silentMode_ = silentMode;
+    detIndex_ = detIndex;
+    numUnitsPerDetector_ = numUnitsPerDetector;
+    udpPortNumber_ = udpPortNumber;
+
+    switch (dynamicRange_) {
+    case 16:
+        dataType_ = PredType::STD_U16LE;
+        break;
+    case 32:
+        dataType_ = PredType::STD_U32LE;
+        break;
+    default:
+        dataType_ = PredType::STD_U8LE;
+        break;
+    }
+
+    CreateFile();
 }
 
-void HDF5DataFile::CreateDataFile(std::string filePath,
+void HDF5DataFile::CreateFile() {
-                                  std::string fileNamePrefix,
+
-                                  uint64_t fileIndex, bool overWriteEnable,
+    numFilesInAcquisition_++;
-                                  bool silentMode, int detIndex,
-                                  int numUnitsPerDetector,
-                                  uint32_t udpPortNumber) {
 
     std::ostringstream os;
-    os << filePath << "/" << fileNamePrefix << "_d"
+    os << filePath_ << "/" << fileNamePrefix_ << "_d"
-       << (detIndex * numUnitsPerDetector + index_) << "_f" << subFileIndex_
+       << (detIndex_ * numUnitsPerDetector_ + index_) << "_f" << subFileIndex_
-       << '_' << fileIndex << ".h5";
+       << '_' << fileIndex_ << ".h5";
     fileName_ = os.str();
 
     std::lock_guard<std::mutex> lock(*hdf5Lib_);
@@ -115,7 +141,7 @@ void HDF5DataFile::CreateDataFile(std::string filePath,
         FileAccPropList fapl;
         fapl.setFcloseDegree(H5F_CLOSE_STRONG);
         fd_ = nullptr;
-        if (!overWriteEnable)
+        if (!overWriteEnable_)
             fd_ = new H5File(fileName_.c_str(), H5F_ACC_EXCL,
                              FileCreatPropList::DEFAULT, fapl);
         else
@@ -146,14 +172,14 @@ void HDF5DataFile::CreateDataFile(std::string filePath,
         // fill value
         DSetCreatPropList plist;
         int fill_value = -1;
-        plist.setFillValue(dataType, &fill_value);
+        plist.setFillValue(dataType_, &fill_value);
         // always create chunked dataset as unlimited is only
         // supported with chunked layout
         hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
         plist.setChunk(3, chunk_dims);
         dataSet_ = nullptr;
-        dataSet_ = new DataSet(
+        dataSet_ = new DataSet(fd_->createDataSet(dsetname.c_str(), dataType_,
-            fd_->createDataSet(dsetname.c_str(), dataType, *dataSpace_, plist));
+                                                  *dataSpace_, plist));
 
         // create parameter datasets
         hsize_t dims[1] = {nDimx};
@@ -179,8 +205,175 @@ void HDF5DataFile::CreateDataFile(std::string filePath,
         throw sls::RuntimeError("Could not create HDF5 handles in object " +
                                 index_);
     }
-    if (!(silentMode)) {
+    if (!silentMode_) {
-        LOG(logINFO) << "[" << udpPortNumber
+        LOG(logINFO) << "[" << udpPortNumber_
                      << "]: HDF5 File created: " << fileName_;
     }
 }
+
+void HDF5DataFile::WriteToFile(char *buffer, int bufferSize,
+                               uint64_t currentFrameNumber,
+                               uint32_t numPacketsCaught) {
+
+    // check if maxframesperfile = 0 for infinite
+    if (maxFramesPerFile_ && (numFramesInFile_ >= maxFramesPerFile_)) {
+        CloseFile();
+        ++subFileIndex_;
+        CreateFile();
+    }
+    numFramesInFile_++;
+
+    // extend dataset (when receiver start followed by many status starts
+    // (jungfrau)))
+    if (currentFrameNumber >= extNumImages_) {
+        ExtendDataset();
+    }
+
+    WriteDataFile(currentFrameNumber, buffer + sizeof(sls_receiver_header));
+    WriteParameterDatasets(currentFrameNumber, (sls_receiver_header *)(buffer));
+}
+
+void HDF5DataFile::WriteDataFile(uint64_t currentFrameNumber, char *buffer) {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+
+    uint64_t nDimx =
+        ((maxFramesPerFile_ == 0) ? currentFrameNumber
+                                  : currentFrameNumber % maxFramesPerFile_);
+    uint32_t nDimy = nPixelsY_;
+    uint32_t nDimz = ((dynamicRange_ == 4) ? (nPixelsX_ / 2) : nPixelsX_);
+
+    hsize_t count[3] = {1, nDimy, nDimz};
+    hsize_t start[3] = {nDimx, 0, 0};
+    hsize_t dims2[2] = {nDimy, nDimz};
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        dataSpace_->selectHyperslab(H5S_SELECT_SET, count, start);
+        DataSpace memspace(2, dims2);
+        dataSet_->write(buffer, dataType_, memspace, *dataSpace_);
+        memspace.close();
+    } catch (const Exception &error) {
+        LOG(logERROR) << "Could not write to file in object " << index_;
+        error.printErrorStack();
+        throw sls::RuntimeError("Could not write to file in object " +
+                                std::to_string(index_));
+    }
+}
+
+void HDF5DataFile::WriteParameterDatasets(uint64_t currentFrameNumber,
+                                          sls_receiver_header *rheader) {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+
+    uint64_t fnum =
+        ((maxFramesPerFile_ == 0) ? currentFrameNumber
+                                  : currentFrameNumber % maxFramesPerFile_);
+
+    sls_detector_header header = rheader->detHeader;
+    hsize_t count[1] = {1};
+    hsize_t start[1] = {fnum};
+    int i = 0;
+    try {
+        Exception::dontPrint(); // to handle errors
+        dataSpacePara_->selectHyperslab(H5S_SELECT_SET, count, start);
+        DataSpace memspace(H5S_SCALAR);
+        dataSetPara_[0]->write(&header.frameNumber, parameterDataTypes_[0],
+                               memspace, *dataSpacePara_);
+        i = 1;
+        dataSetPara_[1]->write(&header.expLength, parameterDataTypes_[1],
+                               memspace, *dataSpacePara_);
+        i = 2;
+        dataSetPara_[2]->write(&header.packetNumber, parameterDataTypes_[2],
+                               memspace, *dataSpacePara_);
+        i = 3;
+        dataSetPara_[3]->write(&header.bunchId, parameterDataTypes_[3],
+                               memspace, *dataSpacePara_);
+        i = 4;
+        dataSetPara_[4]->write(&header.timestamp, parameterDataTypes_[4],
+                               memspace, *dataSpacePara_);
+        i = 5;
+        dataSetPara_[5]->write(&header.modId, parameterDataTypes_[5], memspace,
+                               *dataSpacePara_);
+        i = 6;
+        dataSetPara_[6]->write(&header.row, parameterDataTypes_[6], memspace,
+                               *dataSpacePara_);
+        i = 7;
+        dataSetPara_[7]->write(&header.column, parameterDataTypes_[7], memspace,
+                               *dataSpacePara_);
+        i = 8;
+        dataSetPara_[8]->write(&header.reserved, parameterDataTypes_[8],
+                               memspace, *dataSpacePara_);
+        i = 9;
+        dataSetPara_[9]->write(&header.debug, parameterDataTypes_[9], memspace,
+                               *dataSpacePara_);
+        i = 10;
+        dataSetPara_[10]->write(&header.roundRNumber, parameterDataTypes_[10],
+                                memspace, *dataSpacePara_);
+        i = 11;
+        dataSetPara_[11]->write(&header.detType, parameterDataTypes_[11],
+                                memspace, *dataSpacePara_);
+        i = 12;
+        dataSetPara_[12]->write(&header.version, parameterDataTypes_[12],
+                                memspace, *dataSpacePara_);
+        i = 13;
+
+        // contiguous bitset
+        if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
+            dataSetPara_[13]->write((char *)&(rheader->packetsMask),
+                                    parameterDataTypes_[13], memspace,
+                                    *dataSpacePara_);
+        }
+
+        // not contiguous bitset
+        else {
+            // get contiguous representation of bit mask
+            bitset_storage storage;
+            memset(storage, 0, sizeof(bitset_storage));
+            sls_bitset bits = rheader->packetsMask;
+            for (int i = 0; i < MAX_NUM_PACKETS; ++i)
+                storage[i >> 3] |= (bits[i] << (i & 7));
+            // write bitmask
+            dataSetPara_[13]->write((char *)storage, parameterDataTypes_[13],
+                                    memspace, *dataSpacePara_);
+        }
+        i = 14;
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        throw sls::RuntimeError(
+            "Could not write parameters (index:" + std::to_string(i) +
+            ") to file in object " + std::to_string(index_));
+    }
+}
+
+void HDF5DataFile::ExtendDataset() {
+    std::lock_guard<std::mutex> lock(*hdf5Lib_);
+
+    try {
+        Exception::dontPrint(); // to handle errors
+
+        hsize_t dims[3];
+        dataSpace_->getSimpleExtentDims(dims);
+        dims[0] += numImages_;
+
+        dataSet_->extend(dims);
+        delete dataSpace_;
+        dataSpace_ = nullptr;
+        dataSpace_ = new DataSpace(dataSet_->getSpace());
+
+        hsize_t dims_para[1] = {dims[0]};
+        for (unsigned int i = 0; i < dataSetPara_.size(); ++i)
+            dataSetPara_[i]->extend(dims_para);
+        delete dataSpacePara_;
+        dataSpacePara_ = nullptr;
+        dataSpacePara_ = new DataSpace(dataSetPara_[0]->getSpace());
+
+    } catch (const Exception &error) {
+        error.printErrorStack();
+        throw sls::RuntimeError("Could not extend dataset in object " +
+                                std::to_string(index_));
+    }
+    if (!silentMode_) {
+        LOG(logINFO) << index_ << " Extending HDF5 dataset by " << extNumImages_
+                     << ", Total x Dimension: " << (extNumImages_ + numImages_);
+    }
+    extNumImages_ += numImages_;
+}

slsReceiverSoftware/src/HDF5DataFile.h

@@ -15,40 +15,58 @@ class HDF5DataFile : private virtual slsDetectorDefs, public File {
     HDF5DataFile(int index, std::mutex *hdf5Lib);
     ~HDF5DataFile();
 
+    uint32_t GetFilesInAcquisition() override;
+
     void CloseFile() override;
 
-    void CreateFirstDataFile(std::string filePath, std::string fileNamePrefix,
+    void CreateFirstHDF5DataFile(std::string filePath,
-                             uint64_t fileIndex, bool overWriteEnable,
+                                 std::string fileNamePrefix, uint64_t fileIndex,
-                             bool silentMode, int detIndex,
+                                 bool overWriteEnable, bool silentMode,
-                             int numUnitsPerDetector, uint32_t udpPortNumber,
+                                 int detIndex, int numUnitsPerDetector,
+                                 uint32_t udpPortNumber,
                                  uint32_t maxFramesPerFile, uint64_t numImages,
                                  uint32_t nPIxelsX, uint32_t nPIxelsY,
                                  uint32_t dynamicRange) override;
 
-    void CreateDataFile(std::string filePath, std::string fileNamePrefix,
+    void WriteToFile(char *buffer, int buffersize, uint64_t currentFrameNumber,
-                        uint64_t fileIndex, bool overWriteEnable,
+                     uint32_t numPacketsCaught) override;
-                        bool silentMode, int detIndex, int numUnitsPerDetector,
-                        uint32_t udpPortNumber) override;
 
   private:
+    void CreateFile();
+    void WriteDataFile(uint64_t currentFrameNumber, char *buffer);
+    void WriteParameterDatasets(uint64_t currentFrameNumber,
+                                sls_receiver_header *rheader);
+    void ExtendDataset();
+
     int index_;
     std::mutex *hdf5Lib_;
     H5File *fd_{nullptr};
     std::string fileName_;
     DataSpace *dataSpace_{nullptr};
     DataSet *dataSet_{nullptr};
-    DataType dataType{PredType::STD_U16LE};
+    DataType dataType_{PredType::STD_U16LE};
 
     DataSpace *dataSpacePara_{nullptr};
     std::vector<DataSet *> dataSetPara_{nullptr};
     std::vector<std::string> parameterNames_;
     std::vector<DataType> parameterDataTypes_;
 
-    uint32_t subFileIndex_{0}; // to do reset in createfirstdatafile
+    uint32_t subFileIndex_{0};
+    uint32_t numFramesInFile_{0};
+    uint32_t numFilesInAcquisition_{0};
     uint32_t maxFramesPerFile_{0};
     uint64_t numImages_{0};
-    uint64_t extNumImages_{0}; // TODO needs to be defined in startofacquisition
+    uint64_t extNumImages_{0};
     uint32_t nPixelsX_{0};
     uint32_t nPixelsY_{0};
     uint32_t dynamicRange_{0};
+
+    std::string filePath_;
+    std::string fileNamePrefix_;
+    uint64_t fileIndex_{0};
+    bool overWriteEnable_{false};
+    bool silentMode_{false};
+    int detIndex_{0};
+    int numUnitsPerDetector_{0};
+    uint32_t udpPortNumber_{0};
 };