wip rewrite

slsReceiverSoftware/CMakeLists.txt

@@ -3,7 +3,8 @@ set(SOURCES
     src/ClientInterface.cpp
     src/Receiver.cpp
     src/File.cpp
-    src/BinaryFile.cpp
+    src/BinaryDataFile.cpp
+    src/BinaryMasterFile.cpp
     src/ThreadObject.cpp
     src/Listener.cpp
     src/DataProcessor.cpp
@@ -22,7 +23,9 @@ if (SLS_USE_HDF5)
 	        -DHDF5C ${HDF5_DEFINITIONS}
 	    )
 	    list (APPEND SOURCES 
-	        src/HDF5File.cpp 
+	        src/HDF5DataFile.cpp 
+	        src/HDF5MasterFile.cpp 
+	        src/HDF5VirtualFile.cpp 
 	    )
 endif (SLS_USE_HDF5)
 

slsReceiverSoftware/src/DataProcessor.cpp

@@ -6,12 +6,15 @@
  ***********************************************/
 
 #include "DataProcessor.h"
-#include "BinaryFile.h"
+#include "BinaryDataFile.h"
+#include "BinaryMasterFile.h"
 #include "Fifo.h"
 #include "GeneralData.h"
 #include "MasterAttributes.h"
 #ifdef HDF5C
-#include "HDF5File.h"
+#include "HDF5DataFile.h"
+#include "HDF5MasterFile.h"
+#include "HDF5VirtualFile.h"
 #endif
 #include "DataStreamer.h"
 #include "sls/sls_detector_exceptions.h"
@@ -121,8 +124,6 @@ void DataProcessor::StopProcessing(char *buf) {
     else
         fifo->FreeAddress(buf);
 
-    if (file != nullptr)
-        file->CloseCurrentFile();
     StopRunning();
     LOG(logDEBUG1) << index << ": Processing Completed";
 }
@@ -189,20 +190,6 @@ uint64_t DataProcessor::ProcessAnImage(char *buf) {
                                 std::string(e.what()));
     }
 
-    // write to file
-    if (file != nullptr) {
-        try {
-            file->WriteToFile(
-                buf + FIFO_HEADER_NUMBYTES,
-                sizeof(sls_receiver_header) +
-                    (uint32_t)(*((uint32_t *)buf)), //+ size of data (resizable
-                                                    // from previous call back
-                fnum - firstIndex, nump);
-        } catch (const sls::RuntimeError &e) {
-            ; // ignore write exception for now (TODO: send error message
-              // via stopReceiver tcp)
-        }
-    }
     return fnum;
 }
 
@@ -246,15 +233,6 @@ bool DataProcessor::CheckCount() {
     return false;
 }
 
-void DataProcessor::SetPixelDimension() {
-    if (file != nullptr) {
-        if (file->GetFileType() == HDF5) {
-            file->SetNumberofPixels(generalData->nPixelsX,
-                                    generalData->nPixelsY);
-        }
-    }
-}
-
 void DataProcessor::registerCallBackRawDataReady(void (*func)(char *, char *,
                                                               uint32_t, void *),
                                                  void *arg) {

slsReceiverSoftware/src/DataProcessor.h

@@ -100,31 +100,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
      */
     void SetGeneralData(GeneralData *g);
 
-    /**
-     * Set up file writer object and call backs
-     * @param ftype file format
-     * @param fwe file write enable
-     * @param act activated
-     * @param depad deactivated padding enable
-     * @param nd pointer to number of detectors in each dimension
-     * @param maxf pointer to max frames per file
-     * @param fname pointer to file name prefix
-     * @param fpath pointer to file path
-     * @param findex pointer to file index
-     * @param owenable pointer to over write enable
-     * @param dindex pointer to detector index
-     * @param nunits pointer to number of threads/ units per detector
-     * @param nf pointer to number of images in acquisition
-     * @param dr pointer to dynamic range
-     * @param portno pointer to udp port number
-     * @param g address of GeneralData (Detector Data) pointer
-     */
-
-    /**
-     * Update pixel dimensions in file writer
-     */
-    void SetPixelDimension();
-
     /**
      * Call back for raw data
      * args to raw data ready callback are

slsReceiverSoftware/src/File.cpp

@@ -1,71 +1,10 @@
-/************************************************
- * @file File.cpp
- * @short sets/gets properties for the file,
- * creates/closes the file and writes data to it
- ***********************************************/
-
 #include "File.h"
 
 #include <iostream>
 
-File::File(int ind, slsDetectorDefs::fileFormat type, uint32_t *maxf, int *nd,
+File::File(int index, slsDetectorDefs::fileFormat type)
-           std::string *fname, std::string *fpath, uint64_t *findex,
+    : index_(index), type_(type) {}
-           bool *owenable, int *dindex, int *nunits, uint64_t *nf, uint32_t *dr,
-           uint32_t *portno, bool *smode)
-    : index(ind), formatType(type), maxFramesPerFile(maxf), numDetX(nd[0]),
-      numDetY(nd[1]), fileNamePrefix(fname), filePath(fpath), fileIndex(findex),
-      overWriteEnable(owenable), detIndex(dindex), numUnitsPerDetector(nunits),
-      numImages(nf), dynamicRange(dr), udpPortNumber(portno), silentMode(smode)
-
-{
-    master = ((index == 0) && (*detIndex == 0)) ? true : false;
-}
 
 File::~File() {}
 
-slsDetectorDefs::fileFormat File::GetFileType() { return formatType; }
+slsDetectorDefs::fileFormat File::GetFileType() { return type_; }
-
-std::string File::GetCurrentFileName() { return currentFileName; }
-
-void File::resetSubFileIndex() { subFileIndex = 0u; }
-
-void File::PrintMembers(TLogLevel level) {
-    LOG(level) << "\nGeneral Writer Variables:" << std::endl
-               << "Index: " << index << std::endl
-               << "Max Frames Per File: " << *maxFramesPerFile << std::endl
-               << "Number of Detectors in x dir: " << numDetX << std::endl
-               << "Number of Detectors in y dir: " << numDetY << std::endl
-               << "File Name Prefix: " << fileNamePrefix << std::endl
-               << "File Path: " << filePath << std::endl
-               << "File Index: " << *fileIndex << std::endl
-               << "Over Write Enable: " << *overWriteEnable << std::endl
-
-               << "Detector Index: " << *detIndex << std::endl
-               << "Number of Units Per Detector: " << *numUnitsPerDetector
-               << std::endl
-               << "Number of Images in Acquisition: " << *numImages << std::endl
-               << "Dynamic Range: " << *dynamicRange << std::endl
-               << "UDP Port number: " << *udpPortNumber << std::endl
-               << "Master File Name: " << masterFileName << std::endl
-               << "Current File Name: " << currentFileName << std::endl
-               << "Silent Mode: " << *silentMode;
-}
-
-void File::GetMemberPointerValues(int *nd, uint32_t *&maxf, std::string *&fname,
-                                  std::string *&fpath, uint64_t *&findex,
-                                  bool *&owenable, int *&dindex, int *&nunits,
-                                  uint64_t *&nf, uint32_t *&dr,
-                                  uint32_t *&portno) {
-    nd[0] = numDetX;
-    nd[1] = numDetY;
-    maxf = maxFramesPerFile;
-    fname = fileNamePrefix;
-    fpath = filePath;
-    findex = fileIndex;
-    owenable = overWriteEnable;
-    dindex = detIndex;
-    nunits = numUnitsPerDetector;
-    nf = numImages;
-    dr = dynamicRange;
-    portno = udpPortNumber;
-}

slsReceiverSoftware/src/File.h

@@ -1,142 +1,24 @@
 #pragma once
-/************************************************
- * @file File.h
- * @short sets/gets properties for the file,
- * creates/closes the file and writes data to it
- ***********************************************/
-/**
- *@short sets/gets properties for the file, creates/closes the file and writes
- *data to it
- */
 
-#include "receiver_defs.h"
-#include "sls/logger.h"
 #include "sls/sls_detector_defs.h"
 
-#include <string>
-
 struct MasterAttributes;
 
 class File : private virtual slsDetectorDefs {
 
   public:
-    /**
+    File(int index, slsDetectorDefs::fileFormat type);
-     * Constructor
-     * creates the File Writer
-     * @param ind self index
-     * @param type file format type
-     * @param maxf pointer to max frames per file
-     * @param nd pointer to number of detectors in each dimension
-     * @param fname pointer to file name prefix
-     * @param fpath pointer to file path
-     * @param findex pointer to file index
-     * @param owenable pointer to over write enable
-     * @param dindex pointer to detector index
-     * @param nunits pointer to number of theads/ units per detector
-     * @param nf pointer to number of images in acquisition
-     * @param dr pointer to dynamic range
-     * @param portno pointer to udp port number for logging
-     * @param smode pointer to silent mode
-     */
-    File(int ind, slsDetectorDefs::fileFormat type, uint32_t *maxf, int *nd,
-         std::string *fname, std::string *fpath, uint64_t *findex,
-         bool *owenable, int *dindex, int *nunits, uint64_t *nf, uint32_t *dr,
-         uint32_t *portno, bool *smode);
-
     virtual ~File();
+
     fileFormat GetFileType();
-    std::string GetCurrentFileName();
-    void resetSubFileIndex();
-    virtual void PrintMembers(TLogLevel level = logDEBUG1);
 
-    /**
+    virtual void CloseFile() = 0;
-     * Get Member Pointer Values before the object is destroyed
+    virtual void CreateMasterFile(MasterAttributes *attr, std::string filePath,
-     * @param nd pointer to number of detectors in each dimension
+                                  std::string fileNamePrefix,
-     * @param maxf pointer to max frames per file
+                                  uint64_t fileIndex, bool overWriteEnable,
-     * @param fname pointer to file name prefix
+                                  bool silentMode) = 0;
-     * @param fpath pointer to file path
-     * @param findex pointer to file index
-     * @param owenable pointer to over write enable
-     * @param dindex pointer to detector index
-     * @param nunits pointer to number of theads/ units per detector
-     * @param nf pointer to number of images in acquisition
-     * @param dr pointer to dynamic range
-     * @param portno pointer to dynamic range
-     */
-    void GetMemberPointerValues(int *nd, uint32_t *&maxf, std::string *&fname,
-                                std::string *&fpath, uint64_t *&findex,
-                                bool *&owenable, int *&dindex, int *&nunits,
-                                uint64_t *&nf, uint32_t *&dr,
-                                uint32_t *&portno);
-
-    virtual void CreateFile() = 0;
-    virtual void CloseCurrentDataFile() = 0;
-    virtual void CloseAllFiles() = 0;
-    virtual void CloseMasterFile() = 0;
-
-    /**
-     * Write data to file
-     * @param buffer buffer to write from
-     * @param buffersize size of buffer
-     * @param fnum current image number
-     * @param nump number of packets caught
-     */
-    virtual void WriteToFile(char *buffer, int buffersize, uint64_t fnum,
-                             uint32_t nump) = 0;
-
-    /**
-     * Create master file
-     * @param attr master file attributes
-     */
-    virtual void CreateMasterFile(MasterAttributes *attr) = 0;
-
-    // HDf5 specific
-    /**
-     * Set Number of pixels
-     * @param nx number of pixels in x direction
-     * @param ny number of pixels in y direction
-     */
-    virtual void SetNumberofPixels(uint32_t nx, uint32_t ny) {
-        LOG(logERROR) << "This is a generic function SetNumberofPixels that "
-                         "should be overloaded by a derived class";
-    }
-
-    /**
-     * Start of Acquisition
-     */
-    virtual void StartofAcquisition() {
-        LOG(logERROR) << "This is a generic function StartofAcquisition that "
-                         "should be overloaded by a derived class";
-    }
-
-    /**
-     * End of Acquisition
-     * @param anyPacketsCaught true if any packets are caught, else false
-     * @param numf number of images caught
-     */
-    virtual void EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
-        LOG(logERROR) << "This is a generic function EndofAcquisition that "
-                         "should be overloaded by a derived class";
-    }
 
   protected:
-    bool master;
+    int index_;
-    int index;
+    slsDetectorDefs::fileFormat type_;
-    slsDetectorDefs::fileFormat formatType;
-    uint32_t *maxFramesPerFile;
-    std::string masterFileName;
-    std::string currentFileName;
-    int numDetX;
-    int numDetY;
-    std::string *fileNamePrefix;
-    std::string *filePath;
-    uint64_t *fileIndex;
-    uint64_t subFileIndex{0};
-    bool *overWriteEnable;
-    int *detIndex;
-    int *numUnitsPerDetector;
-    uint64_t *numImages;
-    uint32_t *dynamicRange;
-    uint32_t *udpPortNumber;
-    bool *silentMode;
 };

slsReceiverSoftware/src/Implementation.cpp

@@ -235,11 +235,11 @@ void Implementation::setModulePositionId(const int id) {
         DEFAULT_ZMQ_RX_PORTNO + (modulePos * (myDetectorType == EIGER ? 2 : 1));
 
     for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-        dataProcessor[i]->SetupFileWriter(
+        /*dataProcessor[i]->SetupFileWriter(
             fileFormatType, fileWriteEnable, masterFileWriteEnable, activated,
             deactivatedPaddingEnable, (int *)numDet, &framesPerFile, &fileName,
             &filePath, &fileIndex, &overwriteEnable, &modulePos, &numThreads,
-            &numberOfTotalFrames, &dynamicRange, &udpPortNum[i], generalData);
+            &numberOfTotalFrames, &dynamicRange, &udpPortNum[i], generalData);*/
     }
     assert(numDet[1] != 0);
     for (unsigned int i = 0; i < listener.size(); ++i) {
@@ -346,13 +346,13 @@ void Implementation::setFileFormat(const fileFormat f) {
         }
         if (fileWriteEnable) {
             for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-                dataProcessor[i]->SetupFileWriter(
+                /*dataProcessor[i]->SetupFileWriter(
                     fileFormatType, fileWriteEnable, masterFileWriteEnable,
                     activated, deactivatedPaddingEnable, (int *)numDet,
                     &framesPerFile, &fileName, &filePath, &fileIndex,
                     &overwriteEnable, &modulePos, &numThreads,
                     &numberOfTotalFrames, &dynamicRange, &udpPortNum[i],
-                    generalData);
+                    generalData);*/
             }
         }
     }
@@ -390,12 +390,12 @@ void Implementation::setFileWriteEnable(const bool b) {
     if (fileWriteEnable != b) {
         fileWriteEnable = b;
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-            dataProcessor[i]->SetupFileWriter(
+            /*dataProcessor[i]->SetupFileWriter(
                 fileFormatType, fileWriteEnable, masterFileWriteEnable,
                 activated, deactivatedPaddingEnable, (int *)numDet,
                 &framesPerFile, &fileName, &filePath, &fileIndex,
                 &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames,
-                &dynamicRange, &udpPortNum[i], generalData);
+                &dynamicRange, &udpPortNum[i], generalData);*/
         }
     }
     LOG(logINFO) << "File Write Enable: "
@@ -410,12 +410,12 @@ void Implementation::setMasterFileWriteEnable(const bool b) {
     if (masterFileWriteEnable != b) {
         masterFileWriteEnable = b;
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-            dataProcessor[i]->SetupFileWriter(
+            /*dataProcessor[i]->SetupFileWriter(
                 fileFormatType, fileWriteEnable, masterFileWriteEnable,
                 activated, deactivatedPaddingEnable, (int *)numDet,
                 &framesPerFile, &fileName, &filePath, &fileIndex,
                 &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames,
-                &dynamicRange, &udpPortNum[i], generalData);
+                &dynamicRange, &udpPortNum[i], generalData);*/
         }
     }
     LOG(logINFO) << "Master File Write Enable: "
@@ -587,7 +587,7 @@ void Implementation::stopReceiver() {
         }
         // to create virtual file & set files/acquisition to 0 (only hdf5 at the
         // moment)
-        dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);
+        /*dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);*/
     }
 
     // wait for the processes (dataStreamer) to be done
@@ -697,14 +697,14 @@ void Implementation::closeFiles() {
     uint64_t maxIndexCaught = 0;
     bool anycaught = false;
     for (const auto &it : dataProcessor) {
-        it->CloseFiles();
+        /*it->CloseFiles();*/
         maxIndexCaught = std::max(maxIndexCaught, it->GetProcessedIndex());
         if (it->GetStartedFlag())
             anycaught = true;
     }
     // to create virtual file & set files/acquisition to 0 (only hdf5 at the
     // moment)
-    dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);
+    /*dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);*/
 }
 
 void Implementation::restreamStop() {
@@ -821,7 +821,7 @@ void Implementation::SetupWriter() {
         masterAttributes->gates = numberOfGates;
         masterAttributes->additionalJsonHeader = additionalJsonHeader;
         try {
-            dataProcessor[0]->CreateMasterFile(masterAttributes.get());
+            /*dataProcessor[0]->CreateMasterFile(masterAttributes.get());*/
         } catch (const sls::RuntimeError &e) {
             shutDownUDPSockets();
             closeFiles();
@@ -833,7 +833,7 @@ void Implementation::SetupWriter() {
     //->startofacquisition(which has all the start, and createfirstdatafile)
     try {
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-            dataProcessor[i]->CreateFirstDataFile();
+            /*dataProcessor[i]->CreateFirstDataFile();*/
         }
     } catch (const sls::RuntimeError &e) {
         shutDownUDPSockets();
@@ -1376,7 +1376,7 @@ void Implementation::setNumberofAnalogSamples(const uint32_t i) {
             readoutType);
 
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
     LOG(logINFO) << "Number of Analog Samples: " << numberOfAnalogSamples;
@@ -1397,7 +1397,7 @@ void Implementation::setNumberofDigitalSamples(const uint32_t i) {
             readoutType);
 
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
     LOG(logINFO) << "Number of Digital Samples: " << numberOfDigitalSamples;
@@ -1419,7 +1419,7 @@ void Implementation::setCounterMask(const uint32_t i) {
                                          tengigaEnable);
         // to update npixelsx, npixelsy in file writer
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
     LOG(logINFO) << "Counter mask: " << sls::ToStringHex(counterMask);
@@ -1444,7 +1444,7 @@ void Implementation::setDynamicRange(const uint32_t i) {
 
             // to update npixelsx, npixelsy in file writer
             for (const auto &it : dataProcessor)
-                it->SetPixelDimension();
+                ; /*it->SetPixelDimension();*/
             fifoDepth = generalData->defaultFifoDepth;
             SetupFifoStructure();
         }
@@ -1463,7 +1463,7 @@ void Implementation::setROI(slsDetectorDefs::ROI arg) {
         generalData->SetROI(arg);
         framesPerFile = generalData->maxFramesPerFile;
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
 
@@ -1551,12 +1551,12 @@ bool Implementation::setActivate(bool enable) {
         activated = enable;
         // disable file writing if deactivated and no padding
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-            dataProcessor[i]->SetupFileWriter(
+            /*dataProcessor[i]->SetupFileWriter(
                 fileFormatType, fileWriteEnable, masterFileWriteEnable,
                 activated, deactivatedPaddingEnable, (int *)numDet,
                 &framesPerFile, &fileName, &filePath, &fileIndex,
                 &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames,
-                &dynamicRange, &udpPortNum[i], generalData);
+                &dynamicRange, &udpPortNum[i], generalData);*/
         }
     }
 
@@ -1573,12 +1573,13 @@ void Implementation::setDeactivatedPadding(bool enable) {
         deactivatedPaddingEnable = enable;
         // disable file writing if deactivated and no padding
         for (unsigned int i = 0; i < dataProcessor.size(); ++i) {
-            dataProcessor[i]->SetupFileWriter(
+            ; /*dataProcessor[i]->SetupFileWriter(
-                fileFormatType, fileWriteEnable, masterFileWriteEnable,
+                 fileFormatType, fileWriteEnable, masterFileWriteEnable,
-                activated, deactivatedPaddingEnable, (int *)numDet,
+                 activated, deactivatedPaddingEnable, (int *)numDet,
-                &framesPerFile, &fileName, &filePath, &fileIndex,
+                 &framesPerFile, &fileName, &filePath, &fileIndex,
-                &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames,
+                 &overwriteEnable, &modulePos, &numThreads,
-                &dynamicRange, &udpPortNum[i], generalData);
+                 &numberOfTotalFrames, &dynamicRange, &udpPortNum[i],
+                 generalData);*/
         }
     }
     LOG(logINFO) << "Deactivated Padding Enable: "
@@ -1622,7 +1623,7 @@ void Implementation::setReadoutMode(const readoutMode f) {
             numberOfAnalogSamples, numberOfDigitalSamples, tengigaEnable,
             readoutType);
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
     LOG(logINFO) << "Readout Mode: " << sls::ToString(f);
@@ -1642,7 +1643,7 @@ void Implementation::setADCEnableMask(uint32_t mask) {
             readoutType);
 
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
     LOG(logINFO) << "ADC Enable Mask for 1Gb mode: 0x" << std::hex
@@ -1664,7 +1665,7 @@ void Implementation::setTenGigaADCEnableMask(uint32_t mask) {
             readoutType);
 
         for (const auto &it : dataProcessor)
-            it->SetPixelDimension();
+            ; /*it->SetPixelDimension();*/
         SetupFifoStructure();
     }
     LOG(logINFO) << "ADC Enable Mask for 10Gb mode: 0x" << std::hex