master hdf5 does not work, but so far good

This commit is contained in:
Dhanya Maliakal
2017-02-16 12:48:15 +01:00
parent 54f92fb26c
commit 2cd38fefcd
14 changed files with 958 additions and 225 deletions

View File

@ -5,13 +5,15 @@
***********************************************/
#ifndef BINARY_FILE_H
#define BINARY_FILE_H
#include "File.h"
/**
*@short sets/gets properties for the binary file, creates/closes the file and writes data to it
*/
#include "File.h"
#include <string>
class BinaryFile : private virtual slsReceiverDefs, public File {
public:
@ -26,10 +28,12 @@ class BinaryFile : private virtual slsReceiverDefs, public File {
* @param owenable pointer to over write enable
* @param dindex pointer to detector index
* @param nunits pointer to number of theads/ units per detector
* @param maxf pointer to max frames per file
* @param nf pointer to number of frames
* @param dr dynamic range
* @param maxf max frames per file
*/
BinaryFile(int ind, char* fname, char* fpath, uint64_t* findex,
bool* frindexenable, bool* owenable, int* dindex, int* nunits, uint32_t maxf);
bool* frindexenable, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, uint32_t maxf);
/**
* Destructor
@ -55,9 +59,23 @@ class BinaryFile : private virtual slsReceiverDefs, public File {
int CreateFile(uint64_t fnum);
/**
* Close File
* Close Current File
*/
void CloseFile();
void CloseCurrentFile();
/**
* Close all Files
*/
void CloseAllFiles();
/**
* Write data to file
* @param buffer buffer to write from
* @param buffersize size of buffer
* @param fnum current image number
* @returns OK or FAIL
*/
int WriteToFile(char* buffer, int buffersize, uint64_t fnum);
/**
* Create File Name in format fpath/fnameprefix_fx_dy_z.raw,
@ -77,18 +95,41 @@ class BinaryFile : private virtual slsReceiverDefs, public File {
/**
* Create File
* @param fd file pointer
* @param owenable overwrite enable
* @param fname complete file name
* @returns OK or FAIL
*/
static int CreateDataFile(bool owenable, char* fname);
static int CreateDataFile(FILE*& fd, bool owenable, std::string fname);
/**
* Close File
* @param fd file pointer
*/
static void CloseDataFile();
static void CloseDataFile(FILE*& fd);
/**
* Write data to file
* @param fd file pointer
* @param buf buffer to write from
* @param bsize size of buffer
* @param fnum current image number
* @returns number of elements written
*/
static int WriteDataFile(FILE* fd, char* buf, int bsize, uint64_t fnum);
/**
* Create common files
* @param en ten giga enable
* @param size image size
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
* @param at acquisition time
* @param ap acquisition period
* @returns OK or FAIL
*/
int CreateCommonFiles(bool en, uint32_t size,
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap);
private:
@ -97,11 +138,47 @@ class BinaryFile : private virtual slsReceiverDefs, public File {
* Get Type
* @return type
*/
fileFormat GetType();
fileFormat GetFileType();
/** Maximum frames per file */
uint32_t maxFramesPerFile;
/** File Descriptor */
FILE* filefd;
/** Master File Descriptor */
static FILE* masterfd;
/**
* Create file names for master and virtual file
* @param m master file name
* @param fpath file path
* @param fnameprefix file name prefix (includes scan and position variables)
* @param findex file index
*/
void CreateCommonFileNames(std::string& m, char* fpath, char* fnameprefix, uint64_t findex);
/*
* Close master and virtual files
*/
void CloseCommonDataFiles();
/**
* Create master and virtual files
* @param m master file name
* @param owenable overwrite enable
* @param tengigaEnable ten giga enable
* @param imageSize image size
* @param nPixelsX number of pixels in x direction
* @param nPixelsY number of pixels in y direction
* @param acquisitionTime acquisition time
* @param acquisitionPeriod acquisition period
* @returns OK or FAIL
*/
int CreateCommonDataFiles(std::string m, bool owenable,
bool tengigaEnable, uint32_t imageSize, uint32_t nPixelsX, uint32_t nPixelsY,
uint64_t acquisitionTime, uint64_t acquisitionPeriod);
};

View File

@ -58,13 +58,6 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
*/
static uint64_t GetRunningMask();
/**
* Set GeneralData pointer to the one given
* @param g address of GeneralData (Detector Data) pointer
*/
static void SetGeneralData(GeneralData*& g);
//*** non static functions ***
//*** getters ***
@ -127,9 +120,10 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
void ResetParametersforNewMeasurement();
/**
* Set Max frames per file
* Set GeneralData pointer to the one given
* @param g address of GeneralData (Detector Data) pointer
*/
void SetMaxFramesPerFile();
void SetGeneralData(GeneralData* g);
/**
* Set File Format
@ -146,20 +140,28 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
* @param owenable pointer to over write enable
* @param dindex pointer to detector index
* @param nunits pointer to number of theads/ units per detector
* @param nf pointer to number of images in acquisition
* @param dr dynamic range
* @param g address of GeneralData (Detector Data) pointer
*/
void SetupFileWriter(char* fname, char* fpath, uint64_t* findex,
bool* frindexenable, bool* owenable, int* dindex, int* nunits);
bool* frindexenable, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr, GeneralData* g = 0);
/**
* Create New File
* @param en ten giga enable
* @param nf number of frames
* @param at acquisition time
* @param ap acquisition period
* @returns OK or FAIL
*/
int CreateNewFile();
int CreateNewFile(bool en, uint64_t nf, uint64_t at, uint64_t ap);
/**
* Closes file
* Closes files
*/
void CloseFile();
void CloseFiles();
private:
@ -226,13 +228,14 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
static pthread_mutex_t Mutex;
/** GeneralData (Detector Data) object */
static const GeneralData* generalData;
const GeneralData* generalData;
/** Fifo structure */
Fifo* fifo;
// individual members
/** Aquisition Started flag */
bool acquisitionStartedFlag;

View File

@ -29,9 +29,11 @@ class File : private virtual slsReceiverDefs {
* @param owenable pointer to over write enable
* @param dindex pointer to detector index
* @param nunits pointer to number of theads/ units per detector
* @param nf pointer to number of images in acquisition
* @param dr dynamic range
*/
File(int ind, char* fname, char* fpath, uint64_t* findex,
bool* frindexenable, bool* owenable, int* dindex, int* nunits);
bool* frindexenable, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr);
/**
* Destructor
@ -53,7 +55,7 @@ class File : private virtual slsReceiverDefs {
* Get Type
* @return type
*/
virtual fileFormat GetType() = 0;
virtual fileFormat GetFileType() = 0;
/**
* Get Member Pointer Values before the object is destroyed
@ -64,9 +66,11 @@ class File : private virtual slsReceiverDefs {
* @param owenable pointer to over write enable
* @param dindex pointer to detector index
* @param nunits pointer to number of theads/ units per detector
* @param nf pointer to number of images in acquisition
* @param dr dynamic range
*/
void GetMemberPointerValues(char* fname, char* fpath, uint64_t* findex,
bool* frindexenable, bool* owenable, int* dindex, int* nunits);
void GetMemberPointerValues(char*& fname, char*& fpath, uint64_t*& findex,
bool*& frindexenable, bool*& owenable, int*& dindex, int*& nunits, uint64_t*& nf, uint32_t* dr);
/**
* Create file
@ -74,17 +78,51 @@ class File : private virtual slsReceiverDefs {
* @returns OK or FAIL
*/
virtual int CreateFile(uint64_t fnum){
cprintf(RED,"This is a generic function that should be overloaded by a derived class\n");
cprintf(RED,"This is a generic function CreateFile that should be overloaded by a derived class\n");
return OK;
}
/**
* Close File
* Close Current File
*/
virtual void CloseFile() {
virtual void CloseCurrentFile() {
cprintf(RED,"This is a generic function CloseCurrentFile that should be overloaded by a derived class\n");
}
/**
* Close Files
*/
virtual void CloseAllFiles() {
cprintf(RED,"This is a generic function that should be overloaded by a derived class\n");
}
/**
* Write data to file
* @param buffer buffer to write from
* @param fnum current image number
* @param OK or FAIL
*/
virtual int WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
cprintf(RED,"This is a generic function WriteToFile that should be overloaded by a derived class\n");
return FAIL;
}
/**
* Create common files
* @param en ten giga enable
* @param size image size
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
* @param at acquisition time
* @param ap acquisition period
* @returns OK or FAIL
*/
virtual int CreateCommonFiles(bool en, uint32_t size,
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap) {
cprintf(RED,"This is a generic function CreateCommonFiles that should be overloaded by a derived class\n");
return OK;
}
// Binary specific
/**
@ -92,13 +130,26 @@ class File : private virtual slsReceiverDefs {
* @param maxf maximum frames per file
*/
virtual void SetMaxFramesPerFile(uint32_t maxf) {
cprintf(RED,"This is a generic function that should be overloaded by a derived class\n");
cprintf(RED,"This is a generic function SetMaxFramesPerFile that should be overloaded by a derived class\n");
}
// HDf5 specific
/**
* Set Number of pixels
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
*/
virtual void SetNumberofPixels(uint32_t nx, uint32_t ny) {
cprintf(RED,"This is a generic function SetNumberofPixels that should be overloaded by a derived class\n");
}
protected:
/** master file writer/reader */
bool master;
/** Self Index */
int index;
@ -126,6 +177,12 @@ class File : private virtual slsReceiverDefs {
/** Number of units per detector. Eg. Eiger has 2, others 1 */
int* numUnitsPerDetector;
/** Number of images in acquisition */
uint64_t* numImages;
/** Dynamic Range */
uint32_t* dynamicRange;
/** Current File Name */
std::string currentFileName;
};

View File

@ -21,6 +21,9 @@ public:
/** Number of Pixels in y axis */
uint32_t nPixelsY;
/** Size of header in Packet */
uint32_t headerSizeinPacket;
/** Size of just data in 1 packet (in bytes) */
uint32_t dataSize;
@ -127,6 +130,7 @@ public:
printf("\n\nDetector Data Variables:\n");
printf( "Pixels X: %d\n"
"Pixels Y: %d\n"
"Header Size in Packet: %d\n"
"Data Size: %d\n"
"Packet Size: %d\n"
"Packets per Frame: %d\n"
@ -143,6 +147,7 @@ public:
"Header Packet Size: %d\n",
nPixelsX,
nPixelsY,
headerSizeinPacket,
dataSize,
packetSize,
packetsPerFrame,
@ -169,6 +174,7 @@ class GotthardData : public GeneralData {
GotthardData(){
nPixelsX = 1280;
nPixelsY = 1;
headerSizeinPacket = 4;
dataSize = 1280;
packetSize = 1286;
packetsPerFrame = 2;
@ -177,7 +183,7 @@ class GotthardData : public GeneralData {
frameIndexOffset = 1;
packetIndexMask = 1;
maxFramesPerFile = MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES;
defaultFifoDepth = 25000;
};
@ -192,13 +198,14 @@ class ShortGotthardData : public GeneralData {
ShortGotthardData(){
nPixelsX = 256;
nPixelsY = 1;
headerSizeinPacket = 4;
dataSize = 512;
packetSize = 518;
packetsPerFrame = 1;
imageSize = dataSize*packetsPerFrame;
frameIndexMask = 0xFFFFFFFF;
maxFramesPerFile = SHORT_MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES;
defaultFifoDepth = 25000;
};
@ -209,8 +216,8 @@ class PropixData : public GeneralData {
private:
/** dynamic range for calculating image size */
const static uint32_t dynamicRange = 16;
/**bytes per pixel for calculating image size */
const static uint32_t bytesPerPixel = 2;
public:
@ -218,15 +225,16 @@ class PropixData : public GeneralData {
PropixData(){
nPixelsX = 22;
nPixelsY = 22;
headerSizeinPacket = 4;
dataSize = 1280;
packetSize = 1286;
packetsPerFrame = 2; //not really
imageSize = nPixelsX*nPixelsY*dynamicRange;
imageSize = nPixelsX*nPixelsY*bytesPerPixel;
frameIndexMask = 0xFFFFFFFE;
frameIndexOffset = 1;
packetIndexMask = 1;
maxFramesPerFile = MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES;
defaultFifoDepth = 25000;
};
@ -244,6 +252,7 @@ class Moench02Data : public GeneralData {
Moench02Data(){
nPixelsX = 160;
nPixelsY = 160;
headerSizeinPacket = 4;
dataSize = 1280;
packetSize = 1286;
packetsPerFrame = 40;
@ -252,7 +261,7 @@ class Moench02Data : public GeneralData {
frameIndexOffset = 8;
packetIndexMask = 0xFF;
maxFramesPerFile = MOENCH_MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE;
defaultFifoDepth = 2500;
};
@ -271,33 +280,23 @@ class Moench03Data : public GeneralData {
public:
/** Size of packet header */
const static uint32_t packetHeaderSize = 22;
/** Constructor */
Moench03Data(){
nPixelsX = 400;
nPixelsY = 400;
headerSizeinPacket = 22;
dataSize = 8192;
packetSize = packetHeaderSize + dataSize;
packetSize = headerSizeinPacket + dataSize;
packetsPerFrame = 40;
imageSize = dataSize*packetsPerFrame;
frameIndexMask = 0xFFFFFFFF;
frameIndexOffset = (6+8);
packetIndexMask = 0xFFFFFFFF;
maxFramesPerFile = JFRAU_MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE;
defaultFifoDepth = 2500;
};
/**
* Print all variables
*/
void Print() const {
GeneralData::Print();
printf("Packet Header Size: %d\n",packetHeaderSize);
}
};
@ -312,12 +311,13 @@ class JCTBData : public GeneralData {
JCTBData(){
nPixelsX = 32;
nPixelsY = 128;
headerSizeinPacket = 22;
dataSize = 8192;
packetSize = 8224;
packetSize = headerSizeinPacket + dataSize;
packetsPerFrame = 1;
imageSize = dataSize*packetsPerFrame;
maxFramesPerFile = JFCTB_MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE;
defaultFifoDepth = 2500;
};
@ -354,12 +354,13 @@ private:
JungfrauData(){
nPixelsX = (256*4);
nPixelsY = 256;
headerSizeinPacket = 22;
dataSize = 8192;
packetSize = packetHeaderSize + dataSize;
packetSize = headerSizeinPacket + dataSize;
packetsPerFrame = 128;
imageSize = dataSize*packetsPerFrame;
maxFramesPerFile = JFRAU_MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE;
defaultFifoDepth = 2500;
};
@ -422,16 +423,17 @@ private:
EigerData(){
nPixelsX = (256*2);
nPixelsY = 256;
headerSizeinPacket = 8;
dataSize = 1024;
packetSize = 1040;
packetSize = headerSizeinPacket + dataSize + 8;
packetsPerFrame = 256;
imageSize = dataSize*packetsPerFrame;
frameIndexMask = 0xffffff;
maxFramesPerFile = EIGER_MAX_FRAMES_PER_FILE;
fifoBufferSize = packetSize*packetsPerFrame;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE;
defaultFifoDepth = 100;
footerOffset = packetHeaderSize+dataSize;
footerOffset = headerSizeinPacket + dataSize;
threadsPerReceiver = 2;
headerPacketSize = 48;
};

View File

@ -3,21 +3,30 @@
* @short sets/gets properties for the HDF5 file,
* creates/closes the file and writes data to it
***********************************************/
//#define HDF5C
//#ifdef HDF5C
#ifndef HDF5_FILE_H
#define HDF5_FILE_H
#include "File.h"
/**
*@short sets/gets properties for the HDF5 file, creates/closes the file and writes data to it
*/
#include "File.h"
#include "H5Cpp.h"
#ifndef H5_NO_NAMESPACE
using namespace H5;
#endif
#include <string>
class HDF5File : private virtual slsReceiverDefs, public File {
public:
/**
* Constructor
* creates the File Writer
* @param ind self index
* @param fname pointer to file name prefix
* @param fpath pointer to file path
* @param findex pointer to file index
@ -25,9 +34,14 @@ class HDF5File : private virtual slsReceiverDefs, public File {
* @param owenable pointer to over write enable
* @param dindex pointer to detector index
* @param nunits pointer to number of theads/ units per detector
* @param nf pointer to number of frames
* @param dr dynamic range
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
*/
HDF5File(int ind, char* fname, char* fpath, uint64_t* findex,
bool* frindexenable, bool* owenable, int* dindex, int* nunits);
bool* frindexenable, bool* owenable, int* dindex, int* nunits, uint64_t* nf, uint32_t* dr,
int nx, int ny);
/**
* Destructor
@ -39,6 +53,13 @@ class HDF5File : private virtual slsReceiverDefs, public File {
*/
void PrintMembers();
/**
* Set Number of pixels
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
*/
void SetNumberofPixels(uint32_t nx, uint32_t ny);
/**
* Create file
* @param fnum current frame index to include in file name
@ -47,9 +68,23 @@ class HDF5File : private virtual slsReceiverDefs, public File {
int CreateFile(uint64_t fnum);
/**
* Close File
* Close Current File
*/
void CloseFile();
void CloseCurrentFile();
/**
* Close all Files
*/
void CloseAllFiles();
/**
* Write data to file
* @param buffer buffer to write from
* @param bsize size of buffer (not used)
* @param fnum current image number
* @returns OK or FAIL
*/
int WriteToFile(char* buffer, int bsize, uint64_t fnum);
/**
* Create File Name in format fpath/fnameprefix_fx_dy_z.raw,
@ -69,16 +104,60 @@ class HDF5File : private virtual slsReceiverDefs, public File {
/**
* Create File
* @param ind object index for debugging
* @param owenable overwrite enable
* @param numf number of images
* @param fname complete file name
* @param fnum current image number
* @param nx number of pixels in x dir
* @param ny number of pixels in y dir
* @param dtype data type
* @param fd file pointer
* @param dspace dataspace pointer
* @param dset dataset pointer
* @returns OK or FAIL
*/
static int CreateDataFile(bool owenable, char* fname);
static int CreateDataFile(int ind, bool owenable, uint64_t numf, std::string fname, uint64_t fnum, int nx, int ny,
DataType dtype, H5File*& fd, DataSpace*& dspace, DataSet*& dset);
/**
* Close File
* @param fd file pointer
* @param dp dataspace pointer
* @param ds dataset pointer
*/
static void CloseDataFile();
static void CloseDataFile(H5File*& fd, DataSpace*& dp, DataSet*& ds);
/**
* Write data to file
* @param ind object index for debugging
* @param buf buffer to write from
* @param numImages number of images
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
* @param fnum current image number
* @param dspace dataspace pointer
* @param dset dataset pointer
* @param dtype datatype
* @returns OK or FAIL
*/
static int WriteDataFile(int ind, char* buf, uint64_t numImages, int nx, int ny, uint64_t fnum,
DataSpace* dspace, DataSet* dset, DataType dtype);
/**
* Create common files
* @param en ten giga enable
* @param size image size
* @param nx number of pixels in x direction
* @param ny number of pixels in y direction
* @param at acquisition time
* @param ap acquisition period
* @returns OK or FAIL
*/
int CreateCommonFiles(bool en, uint32_t size,
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap);
private:
@ -86,8 +165,73 @@ class HDF5File : private virtual slsReceiverDefs, public File {
* Get Type
* @return type
*/
fileFormat GetType();
fileFormat GetFileType();
/**
* Updates data type depending on current dynamic range
*/
void UpdateDataType();
/**
* Create file names for master and virtual file
* @param m master file name
* @param v virtual file name
* @param fpath file path
* @param fnameprefix file name prefix (includes scan and position variables)
* @param findex file index
*/
void CreateCommonFileNames(std::string& m, std::string& v, char* fpath, char* fnameprefix, uint64_t findex);
/*
* Close master and virtual files
*/
void CloseCommonDataFiles();
/**
* Create master and virtual files
* @param m master file name
* @param v virtual file name
* @param owenable overwrite enable
* @param tengigaEnable ten giga enable
* @param imageSize image size
* @param nPixelsX number of pixels in x direction
* @param nPixelsY number of pixels in y direction
* @param acquisitionTime acquisition time
* @param acquisitionPeriod acquisition period
* @returns OK or FAIL
*/
int CreateCommonDataFiles(std::string m, std::string v, bool owenable,
bool tengigaEnable, uint32_t imageSize, uint32_t nPixelsX, uint32_t nPixelsY,
uint64_t acquisitionTime, uint64_t acquisitionPeriod);
/** mutex to update static items among objects (threads)*/
static pthread_mutex_t Mutex;
/** Master File handle */
static H5File* masterfd;
/** Virtual File handle */
static H5File* virtualfd;
/** File handle */
H5File* filefd;
/** Dataspace handle */
DataSpace* dataspace;
/** DataSet handle */
DataSet* dataset;
/** Datatype of dataset */
DataType datatype;
/** Number of pixels in x direction */
int nPixelsX;
/** Number of pixels in y direction */
int nPixelsY;
};
//#endif
#endif

View File

@ -238,6 +238,10 @@ class Listener : private virtual slsReceiverDefs, public ThreadObject {
/** Carry over packet buffer */
char* carryOverPacket;
/** Listening buffer for one packet - might be removed when we can peek and eiger fnum is in header */
char* listeningPacket;
};
#endif

View File

@ -234,6 +234,12 @@ private:
*/
void StartRunning();
/**
* Verifies if the callbacks are registered for the callback variable
* @returns OK or FAIL
*/
int VerifyCallBackAction();
//*** Class Members ***

View File

@ -16,6 +16,14 @@
//binary
#define FILE_FRAME_HEADER_SIZE 16
#define FILE_BUFFER_SIZE (16*1024*1024) //16mb
//hdf5
#define MAX_CHUNKED_IMAGES 1
//versions
#define HDF5_WRITER_VERSION 1.0
#define BINARY_WRITER_VERSION 1.0 //1 decimal places
//fifo
#define FIFO_HEADER_NUMBYTES 4