diff --git a/slsReceiverSoftware/CMakeLists.txt b/slsReceiverSoftware/CMakeLists.txt index ad2a958c5..8ed871ab5 100755 --- a/slsReceiverSoftware/CMakeLists.txt +++ b/slsReceiverSoftware/CMakeLists.txt @@ -3,7 +3,8 @@ set(SOURCES src/ClientInterface.cpp src/Receiver.cpp src/File.cpp - src/BinaryFile.cpp + src/BinaryDataFile.cpp + src/BinaryMasterFile.cpp src/ThreadObject.cpp src/Listener.cpp src/DataProcessor.cpp @@ -22,7 +23,9 @@ if (SLS_USE_HDF5) -DHDF5C ${HDF5_DEFINITIONS} ) list (APPEND SOURCES - src/HDF5File.cpp + src/HDF5DataFile.cpp + src/HDF5MasterFile.cpp + src/HDF5VirtualFile.cpp ) endif (SLS_USE_HDF5) diff --git a/slsReceiverSoftware/src/DataProcessor.cpp b/slsReceiverSoftware/src/DataProcessor.cpp index 6a64ca706..2e4e961f5 100644 --- a/slsReceiverSoftware/src/DataProcessor.cpp +++ b/slsReceiverSoftware/src/DataProcessor.cpp @@ -6,12 +6,15 @@ ***********************************************/ #include "DataProcessor.h" -#include "BinaryFile.h" +#include "BinaryDataFile.h" +#include "BinaryMasterFile.h" #include "Fifo.h" #include "GeneralData.h" #include "MasterAttributes.h" #ifdef HDF5C -#include "HDF5File.h" +#include "HDF5DataFile.h" +#include "HDF5MasterFile.h" +#include "HDF5VirtualFile.h" #endif #include "DataStreamer.h" #include "sls/sls_detector_exceptions.h" @@ -121,8 +124,6 @@ void DataProcessor::StopProcessing(char *buf) { else fifo->FreeAddress(buf); - if (file != nullptr) - file->CloseCurrentFile(); StopRunning(); LOG(logDEBUG1) << index << ": Processing Completed"; } @@ -189,20 +190,6 @@ uint64_t DataProcessor::ProcessAnImage(char *buf) { std::string(e.what())); } - // write to file - if (file != nullptr) { - try { - file->WriteToFile( - buf + FIFO_HEADER_NUMBYTES, - sizeof(sls_receiver_header) + - (uint32_t)(*((uint32_t *)buf)), //+ size of data (resizable - // from previous call back - fnum - firstIndex, nump); - } catch (const sls::RuntimeError &e) { - ; // ignore write exception for now (TODO: send error message - // via stopReceiver tcp) - } - } return fnum; } @@ -246,15 +233,6 @@ bool DataProcessor::CheckCount() { return false; } -void DataProcessor::SetPixelDimension() { - if (file != nullptr) { - if (file->GetFileType() == HDF5) { - file->SetNumberofPixels(generalData->nPixelsX, - generalData->nPixelsY); - } - } -} - void DataProcessor::registerCallBackRawDataReady(void (*func)(char *, char *, uint32_t, void *), void *arg) { diff --git a/slsReceiverSoftware/src/DataProcessor.h b/slsReceiverSoftware/src/DataProcessor.h index c49c312fb..791a05b09 100644 --- a/slsReceiverSoftware/src/DataProcessor.h +++ b/slsReceiverSoftware/src/DataProcessor.h @@ -100,31 +100,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject { */ void SetGeneralData(GeneralData *g); - /** - * Set up file writer object and call backs - * @param ftype file format - * @param fwe file write enable - * @param act activated - * @param depad deactivated padding enable - * @param nd pointer to number of detectors in each dimension - * @param maxf pointer to max frames per file - * @param fname pointer to file name prefix - * @param fpath pointer to file path - * @param findex pointer to file index - * @param owenable pointer to over write enable - * @param dindex pointer to detector index - * @param nunits pointer to number of threads/ units per detector - * @param nf pointer to number of images in acquisition - * @param dr pointer to dynamic range - * @param portno pointer to udp port number - * @param g address of GeneralData (Detector Data) pointer - */ - - /** - * Update pixel dimensions in file writer - */ - void SetPixelDimension(); - /** * Call back for raw data * args to raw data ready callback are diff --git a/slsReceiverSoftware/src/File.cpp b/slsReceiverSoftware/src/File.cpp index 047461b75..a06845ace 100644 --- a/slsReceiverSoftware/src/File.cpp +++ b/slsReceiverSoftware/src/File.cpp @@ -1,71 +1,10 @@ -/************************************************ - * @file File.cpp - * @short sets/gets properties for the file, - * creates/closes the file and writes data to it - ***********************************************/ - #include "File.h" #include -File::File(int ind, slsDetectorDefs::fileFormat type, uint32_t *maxf, int *nd, - std::string *fname, std::string *fpath, uint64_t *findex, - bool *owenable, int *dindex, int *nunits, uint64_t *nf, uint32_t *dr, - uint32_t *portno, bool *smode) - : index(ind), formatType(type), maxFramesPerFile(maxf), numDetX(nd[0]), - numDetY(nd[1]), fileNamePrefix(fname), filePath(fpath), fileIndex(findex), - overWriteEnable(owenable), detIndex(dindex), numUnitsPerDetector(nunits), - numImages(nf), dynamicRange(dr), udpPortNumber(portno), silentMode(smode) - -{ - master = ((index == 0) && (*detIndex == 0)) ? true : false; -} +File::File(int index, slsDetectorDefs::fileFormat type) + : index_(index), type_(type) {} File::~File() {} -slsDetectorDefs::fileFormat File::GetFileType() { return formatType; } - -std::string File::GetCurrentFileName() { return currentFileName; } - -void File::resetSubFileIndex() { subFileIndex = 0u; } - -void File::PrintMembers(TLogLevel level) { - LOG(level) << "\nGeneral Writer Variables:" << std::endl - << "Index: " << index << std::endl - << "Max Frames Per File: " << *maxFramesPerFile << std::endl - << "Number of Detectors in x dir: " << numDetX << std::endl - << "Number of Detectors in y dir: " << numDetY << std::endl - << "File Name Prefix: " << fileNamePrefix << std::endl - << "File Path: " << filePath << std::endl - << "File Index: " << *fileIndex << std::endl - << "Over Write Enable: " << *overWriteEnable << std::endl - - << "Detector Index: " << *detIndex << std::endl - << "Number of Units Per Detector: " << *numUnitsPerDetector - << std::endl - << "Number of Images in Acquisition: " << *numImages << std::endl - << "Dynamic Range: " << *dynamicRange << std::endl - << "UDP Port number: " << *udpPortNumber << std::endl - << "Master File Name: " << masterFileName << std::endl - << "Current File Name: " << currentFileName << std::endl - << "Silent Mode: " << *silentMode; -} - -void File::GetMemberPointerValues(int *nd, uint32_t *&maxf, std::string *&fname, - std::string *&fpath, uint64_t *&findex, - bool *&owenable, int *&dindex, int *&nunits, - uint64_t *&nf, uint32_t *&dr, - uint32_t *&portno) { - nd[0] = numDetX; - nd[1] = numDetY; - maxf = maxFramesPerFile; - fname = fileNamePrefix; - fpath = filePath; - findex = fileIndex; - owenable = overWriteEnable; - dindex = detIndex; - nunits = numUnitsPerDetector; - nf = numImages; - dr = dynamicRange; - portno = udpPortNumber; -} +slsDetectorDefs::fileFormat File::GetFileType() { return type_; } diff --git a/slsReceiverSoftware/src/File.h b/slsReceiverSoftware/src/File.h index 0e9fc325c..e2d183a24 100644 --- a/slsReceiverSoftware/src/File.h +++ b/slsReceiverSoftware/src/File.h @@ -1,142 +1,24 @@ #pragma once -/************************************************ - * @file File.h - * @short sets/gets properties for the file, - * creates/closes the file and writes data to it - ***********************************************/ -/** - *@short sets/gets properties for the file, creates/closes the file and writes - *data to it - */ -#include "receiver_defs.h" -#include "sls/logger.h" #include "sls/sls_detector_defs.h" -#include - struct MasterAttributes; class File : private virtual slsDetectorDefs { public: - /** - * Constructor - * creates the File Writer - * @param ind self index - * @param type file format type - * @param maxf pointer to max frames per file - * @param nd pointer to number of detectors in each dimension - * @param fname pointer to file name prefix - * @param fpath pointer to file path - * @param findex pointer to file index - * @param owenable pointer to over write enable - * @param dindex pointer to detector index - * @param nunits pointer to number of theads/ units per detector - * @param nf pointer to number of images in acquisition - * @param dr pointer to dynamic range - * @param portno pointer to udp port number for logging - * @param smode pointer to silent mode - */ - File(int ind, slsDetectorDefs::fileFormat type, uint32_t *maxf, int *nd, - std::string *fname, std::string *fpath, uint64_t *findex, - bool *owenable, int *dindex, int *nunits, uint64_t *nf, uint32_t *dr, - uint32_t *portno, bool *smode); - + File(int index, slsDetectorDefs::fileFormat type); virtual ~File(); + fileFormat GetFileType(); - std::string GetCurrentFileName(); - void resetSubFileIndex(); - virtual void PrintMembers(TLogLevel level = logDEBUG1); - /** - * Get Member Pointer Values before the object is destroyed - * @param nd pointer to number of detectors in each dimension - * @param maxf pointer to max frames per file - * @param fname pointer to file name prefix - * @param fpath pointer to file path - * @param findex pointer to file index - * @param owenable pointer to over write enable - * @param dindex pointer to detector index - * @param nunits pointer to number of theads/ units per detector - * @param nf pointer to number of images in acquisition - * @param dr pointer to dynamic range - * @param portno pointer to dynamic range - */ - void GetMemberPointerValues(int *nd, uint32_t *&maxf, std::string *&fname, - std::string *&fpath, uint64_t *&findex, - bool *&owenable, int *&dindex, int *&nunits, - uint64_t *&nf, uint32_t *&dr, - uint32_t *&portno); - - virtual void CreateFile() = 0; - virtual void CloseCurrentDataFile() = 0; - virtual void CloseAllFiles() = 0; - virtual void CloseMasterFile() = 0; - - /** - * Write data to file - * @param buffer buffer to write from - * @param buffersize size of buffer - * @param fnum current image number - * @param nump number of packets caught - */ - virtual void WriteToFile(char *buffer, int buffersize, uint64_t fnum, - uint32_t nump) = 0; - - /** - * Create master file - * @param attr master file attributes - */ - virtual void CreateMasterFile(MasterAttributes *attr) = 0; - - // HDf5 specific - /** - * Set Number of pixels - * @param nx number of pixels in x direction - * @param ny number of pixels in y direction - */ - virtual void SetNumberofPixels(uint32_t nx, uint32_t ny) { - LOG(logERROR) << "This is a generic function SetNumberofPixels that " - "should be overloaded by a derived class"; - } - - /** - * Start of Acquisition - */ - virtual void StartofAcquisition() { - LOG(logERROR) << "This is a generic function StartofAcquisition that " - "should be overloaded by a derived class"; - } - - /** - * End of Acquisition - * @param anyPacketsCaught true if any packets are caught, else false - * @param numf number of images caught - */ - virtual void EndofAcquisition(bool anyPacketsCaught, uint64_t numf) { - LOG(logERROR) << "This is a generic function EndofAcquisition that " - "should be overloaded by a derived class"; - } + virtual void CloseFile() = 0; + virtual void CreateMasterFile(MasterAttributes *attr, std::string filePath, + std::string fileNamePrefix, + uint64_t fileIndex, bool overWriteEnable, + bool silentMode) = 0; protected: - bool master; - int index; - slsDetectorDefs::fileFormat formatType; - uint32_t *maxFramesPerFile; - std::string masterFileName; - std::string currentFileName; - int numDetX; - int numDetY; - std::string *fileNamePrefix; - std::string *filePath; - uint64_t *fileIndex; - uint64_t subFileIndex{0}; - bool *overWriteEnable; - int *detIndex; - int *numUnitsPerDetector; - uint64_t *numImages; - uint32_t *dynamicRange; - uint32_t *udpPortNumber; - bool *silentMode; + int index_; + slsDetectorDefs::fileFormat type_; }; diff --git a/slsReceiverSoftware/src/Implementation.cpp b/slsReceiverSoftware/src/Implementation.cpp index da7d972b1..f3162ba5d 100644 --- a/slsReceiverSoftware/src/Implementation.cpp +++ b/slsReceiverSoftware/src/Implementation.cpp @@ -235,11 +235,11 @@ void Implementation::setModulePositionId(const int id) { DEFAULT_ZMQ_RX_PORTNO + (modulePos * (myDetectorType == EIGER ? 2 : 1)); for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->SetupFileWriter( + /*dataProcessor[i]->SetupFileWriter( fileFormatType, fileWriteEnable, masterFileWriteEnable, activated, deactivatedPaddingEnable, (int *)numDet, &framesPerFile, &fileName, &filePath, &fileIndex, &overwriteEnable, &modulePos, &numThreads, - &numberOfTotalFrames, &dynamicRange, &udpPortNum[i], generalData); + &numberOfTotalFrames, &dynamicRange, &udpPortNum[i], generalData);*/ } assert(numDet[1] != 0); for (unsigned int i = 0; i < listener.size(); ++i) { @@ -346,13 +346,13 @@ void Implementation::setFileFormat(const fileFormat f) { } if (fileWriteEnable) { for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->SetupFileWriter( + /*dataProcessor[i]->SetupFileWriter( fileFormatType, fileWriteEnable, masterFileWriteEnable, activated, deactivatedPaddingEnable, (int *)numDet, &framesPerFile, &fileName, &filePath, &fileIndex, &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames, &dynamicRange, &udpPortNum[i], - generalData); + generalData);*/ } } } @@ -390,12 +390,12 @@ void Implementation::setFileWriteEnable(const bool b) { if (fileWriteEnable != b) { fileWriteEnable = b; for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->SetupFileWriter( + /*dataProcessor[i]->SetupFileWriter( fileFormatType, fileWriteEnable, masterFileWriteEnable, activated, deactivatedPaddingEnable, (int *)numDet, &framesPerFile, &fileName, &filePath, &fileIndex, &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames, - &dynamicRange, &udpPortNum[i], generalData); + &dynamicRange, &udpPortNum[i], generalData);*/ } } LOG(logINFO) << "File Write Enable: " @@ -410,12 +410,12 @@ void Implementation::setMasterFileWriteEnable(const bool b) { if (masterFileWriteEnable != b) { masterFileWriteEnable = b; for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->SetupFileWriter( + /*dataProcessor[i]->SetupFileWriter( fileFormatType, fileWriteEnable, masterFileWriteEnable, activated, deactivatedPaddingEnable, (int *)numDet, &framesPerFile, &fileName, &filePath, &fileIndex, &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames, - &dynamicRange, &udpPortNum[i], generalData); + &dynamicRange, &udpPortNum[i], generalData);*/ } } LOG(logINFO) << "Master File Write Enable: " @@ -587,7 +587,7 @@ void Implementation::stopReceiver() { } // to create virtual file & set files/acquisition to 0 (only hdf5 at the // moment) - dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught); + /*dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);*/ } // wait for the processes (dataStreamer) to be done @@ -697,14 +697,14 @@ void Implementation::closeFiles() { uint64_t maxIndexCaught = 0; bool anycaught = false; for (const auto &it : dataProcessor) { - it->CloseFiles(); + /*it->CloseFiles();*/ maxIndexCaught = std::max(maxIndexCaught, it->GetProcessedIndex()); if (it->GetStartedFlag()) anycaught = true; } // to create virtual file & set files/acquisition to 0 (only hdf5 at the // moment) - dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught); + /*dataProcessor[0]->EndofAcquisition(anycaught, maxIndexCaught);*/ } void Implementation::restreamStop() { @@ -821,7 +821,7 @@ void Implementation::SetupWriter() { masterAttributes->gates = numberOfGates; masterAttributes->additionalJsonHeader = additionalJsonHeader; try { - dataProcessor[0]->CreateMasterFile(masterAttributes.get()); + /*dataProcessor[0]->CreateMasterFile(masterAttributes.get());*/ } catch (const sls::RuntimeError &e) { shutDownUDPSockets(); closeFiles(); @@ -833,7 +833,7 @@ void Implementation::SetupWriter() { //->startofacquisition(which has all the start, and createfirstdatafile) try { for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->CreateFirstDataFile(); + /*dataProcessor[i]->CreateFirstDataFile();*/ } } catch (const sls::RuntimeError &e) { shutDownUDPSockets(); @@ -1376,7 +1376,7 @@ void Implementation::setNumberofAnalogSamples(const uint32_t i) { readoutType); for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } LOG(logINFO) << "Number of Analog Samples: " << numberOfAnalogSamples; @@ -1397,7 +1397,7 @@ void Implementation::setNumberofDigitalSamples(const uint32_t i) { readoutType); for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } LOG(logINFO) << "Number of Digital Samples: " << numberOfDigitalSamples; @@ -1419,7 +1419,7 @@ void Implementation::setCounterMask(const uint32_t i) { tengigaEnable); // to update npixelsx, npixelsy in file writer for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } LOG(logINFO) << "Counter mask: " << sls::ToStringHex(counterMask); @@ -1444,7 +1444,7 @@ void Implementation::setDynamicRange(const uint32_t i) { // to update npixelsx, npixelsy in file writer for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ fifoDepth = generalData->defaultFifoDepth; SetupFifoStructure(); } @@ -1463,7 +1463,7 @@ void Implementation::setROI(slsDetectorDefs::ROI arg) { generalData->SetROI(arg); framesPerFile = generalData->maxFramesPerFile; for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } @@ -1551,12 +1551,12 @@ bool Implementation::setActivate(bool enable) { activated = enable; // disable file writing if deactivated and no padding for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->SetupFileWriter( + /*dataProcessor[i]->SetupFileWriter( fileFormatType, fileWriteEnable, masterFileWriteEnable, activated, deactivatedPaddingEnable, (int *)numDet, &framesPerFile, &fileName, &filePath, &fileIndex, &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames, - &dynamicRange, &udpPortNum[i], generalData); + &dynamicRange, &udpPortNum[i], generalData);*/ } } @@ -1573,12 +1573,13 @@ void Implementation::setDeactivatedPadding(bool enable) { deactivatedPaddingEnable = enable; // disable file writing if deactivated and no padding for (unsigned int i = 0; i < dataProcessor.size(); ++i) { - dataProcessor[i]->SetupFileWriter( - fileFormatType, fileWriteEnable, masterFileWriteEnable, - activated, deactivatedPaddingEnable, (int *)numDet, - &framesPerFile, &fileName, &filePath, &fileIndex, - &overwriteEnable, &modulePos, &numThreads, &numberOfTotalFrames, - &dynamicRange, &udpPortNum[i], generalData); + ; /*dataProcessor[i]->SetupFileWriter( + fileFormatType, fileWriteEnable, masterFileWriteEnable, + activated, deactivatedPaddingEnable, (int *)numDet, + &framesPerFile, &fileName, &filePath, &fileIndex, + &overwriteEnable, &modulePos, &numThreads, + &numberOfTotalFrames, &dynamicRange, &udpPortNum[i], + generalData);*/ } } LOG(logINFO) << "Deactivated Padding Enable: " @@ -1622,7 +1623,7 @@ void Implementation::setReadoutMode(const readoutMode f) { numberOfAnalogSamples, numberOfDigitalSamples, tengigaEnable, readoutType); for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } LOG(logINFO) << "Readout Mode: " << sls::ToString(f); @@ -1642,7 +1643,7 @@ void Implementation::setADCEnableMask(uint32_t mask) { readoutType); for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } LOG(logINFO) << "ADC Enable Mask for 1Gb mode: 0x" << std::hex @@ -1664,7 +1665,7 @@ void Implementation::setTenGigaADCEnableMask(uint32_t mask) { readoutType); for (const auto &it : dataProcessor) - it->SetPixelDimension(); + ; /*it->SetPixelDimension();*/ SetupFifoStructure(); } LOG(logINFO) << "ADC Enable Mask for 10Gb mode: 0x" << std::hex