detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-06-12 21:07:13 +02:00
Code Issues Actions Packages Releases Activity

created virtual datasets for bunchid and subframenumber

Browse Source
This commit is contained in:
Dhanya Maliakal
2017-03-02 11:43:11 +01:00
parent a7309be567
commit 537260879a
9 changed files with 431 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
slsReceiverSoftware/include/DataStreamer.h
View File

@ -82,7 +82,7 @@ class DataStreamer : private virtual slsReceiverDefs, public ThreadObject {
/**
* Reset parameters for new measurement (eg. for each scan)
*/
void ResetParametersforNewMeasurement();
void ResetParametersforNewMeasurement(char* fname);
/**
* Set GeneralData pointer to the one given
@ -237,5 +237,8 @@ class DataStreamer : private virtual slsReceiverDefs, public ThreadObject {
/** Frame Number of First Frame for each real time acquisition (eg. for each scan) */
uint64_t firstMeasurementIndex;
/* File name to stream */
char fileNametoStream[MAX_STR_LENGTH];
};

24
slsReceiverSoftware/include/GeneralData.h
View File

@ -94,13 +94,14 @@ public:
* Get Header Infomation (frame number, packet number)
* @param index thread index for debugging purposes
* @param packetData pointer to data
* @param dynamicRange dynamic range to assign subframenumber if 32 bit mode
* @param frameNumber frame number
* @param packetNumber packet number
* @param subFrameNumber sub frame number if applicable
* @param bunchId bunch id
*/
virtual void GetHeaderInfo(int index, char* packetData, uint64_t& frameNumber,
uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t bunchId) const
virtual void GetHeaderInfo(int index, char* packetData, uint32_t dynamicRange,
uint64_t& frameNumber, uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t& bunchId) const
{
subFrameNumber = -1;
bunchId = -1;
@ -388,13 +389,14 @@ private:
* Get Header Infomation (frame number, packet number)
* @param index thread index for debugging purposes
* @param packetData pointer to data
* @param dynamicRange dynamic range to assign subframenumber if 32 bit mode
* @param frameNumber frame number
* @param packetNumber packet number
* @param subFrameNumber sub frame number if applicable
* @param bunchId bunch id
*/
void GetHeaderInfo(int index, char* packetData, uint64_t& frameNumber,
uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t bunchId) const
void GetHeaderInfo(int index, char* packetData, uint32_t dynamicRange,
uint64_t& frameNumber, uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t& bunchId) const
{
subFrameNumber = -1;
jfrau_packet_header_t* header = (jfrau_packet_header_t*)(packetData);
@ -449,7 +451,7 @@ private:
packetsPerFrame = 256;
imageSize = dataSize*packetsPerFrame;
frameIndexMask = 0xffffff;
maxFramesPerFile = 5;//EIGER_MAX_FRAMES_PER_FILE;
maxFramesPerFile = EIGER_MAX_FRAMES_PER_FILE;
fifoBufferSize = imageSize;
fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE;
defaultFifoDepth = 100;
@ -475,19 +477,23 @@ private:
* Get Header Infomation (frame number, packet number)
* @param index thread index for debugging purposes
* @param packetData pointer to data
* @param dynamicRange dynamic range to assign subframenumber if 32 bit mode
* @param frameNumber frame number
* @param packetNumber packet number
* @param subFrameNumber sub frame number if applicable
* @param bunchId bunch id
*/
void GetHeaderInfo(int index, char* packetData, uint64_t& frameNumber,
uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t bunchId) const {
void GetHeaderInfo(int index, char* packetData, uint32_t dynamicRange,
uint64_t& frameNumber, uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t& bunchId) const {
bunchId = -1;
subFrameNumber = -1;
eiger_packet_footer_t* footer = (eiger_packet_footer_t*)(packetData + footerOffset);
frameNumber = (uint64_t)((*( (uint64_t*) footer)) & frameIndexMask);
packetNumber = (uint32_t)(*( (uint16_t*) footer->packetNumber))-1;
eiger_packet_header_t* header = (eiger_packet_header_t*) (packetData);
subFrameNumber = (uint64_t) *( (uint32_t*) header->subFrameNumber);
if (dynamicRange == 32) {
eiger_packet_header_t* header = (eiger_packet_header_t*) (packetData);
subFrameNumber = (uint64_t) *( (uint32_t*) header->subFrameNumber);
}
}
/**

21
slsReceiverSoftware/include/HDF5File.h
View File

@ -144,6 +144,27 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
/** Datatype of dataset */
DataType datatype;
/** Dataspace of parameters */
DataSpace* dataspace_para;
/** parameter1 */
std::string para1;
/** Dataset of parameter1 */
DataSet* dataset_para1;
/** Datatype of parameter1 */
DataType datatype_para1;
/** parameter2 */
std::string para2;
/** Dataset of parameter2 */
DataSet* dataset_para2;
/** Datatype of parameter2 */
DataType datatype_para2;
/** Number of pixels in x direction */
uint32_t nPixelsX;

439
slsReceiverSoftware/include/HDF5FileStatic.h
View File

@ -104,13 +104,16 @@ public:
* @param ds dataset pointer
*/
static void CloseDataFile(int ind, H5File*& fd, DataSpace*& dp, DataSet*& ds)
static void CloseDataFile(int ind, H5File*& fd, DataSpace*& dp, DataSet*& ds,
DataSet*& ds_p1, DataSet*& ds_p2)
{
try {
Exception::dontPrint(); //to handle errors
if(dp) {delete dp; dp = 0;}
if(ds) {delete ds; ds = 0;}
if(fd) {delete fd; fd = 0;}
if(dp) {delete dp; dp = 0;}
if(ds) {delete ds; ds = 0;}
if(ds_p1) {delete ds_p1; ds_p1 = 0;}
if(ds_p2) {delete ds_p2; ds_p2 = 0;}
if(fd) {delete fd; fd = 0;}
} catch(Exception error) {
cprintf(RED,"Error in closing HDF5 handles\n");
error.printError();
@ -182,6 +185,34 @@ public:
}
/**
* Write Parameter Arrays as datasets (to virtual file)
*/
template <typename P1, typename P2>
static int WriteParameterDatasets(int ind, DataSpace* dspace_para, uint64_t fnum,
DataSet*& dset_para1, DataType dtype_para1, P1* para1,
DataSet*& dset_para2, DataType dtype_para2, P2* para2)
{
hsize_t count[1] = {1};
hsize_t start[1] = {fnum};
hsize_t dims2[2] = {1};
try{
Exception::dontPrint(); //to handle errors
dspace_para->selectHyperslab( H5S_SELECT_SET, count, start);
DataSpace memspace(H5S_SCALAR);
dset_para1->write(para1, dtype_para1, memspace, *dspace_para);
dset_para2->write(para2, dtype_para2, memspace, *dspace_para);
}
catch(Exception error){
cprintf(RED,"Error in writing parameters to file in object %d\n",ind);
error.printError();
return 1;
}
return 0;
}
/**
* Create master file
* @param fname master file name
@ -307,12 +338,22 @@ public:
* @param dset dataset pointer
* @param version version of software for hdf5 writing
* @param maxchunkedimages maximum chunked images
* @param dspace_para dataspace of parameters
* @param para1 parameter 1 name
* @param dset_para1 dataset of parameter 1
* @param dtype_para1 datatype of parameter 1
* @param para2 parameter 2 name
* @param dset_para2 dataset of parameter 2
* @param dtype_para2 datatype of parameter 2
* @returns 0 for success and 1 for fail
*/
static int CreateDataFile(int ind, bool owenable, string fname, bool frindexenable,
uint64_t fnum, uint64_t nDimx, uint32_t nDimy, uint32_t nDimz,
DataType dtype, H5File*& fd, DataSpace*& dspace, DataSet*& dset,
double version, uint64_t maxchunkedimages)
double version, uint64_t maxchunkedimages,
DataSpace*& dspace_para,
string para1, DataSet*& dset_para1, DataType dtype_para1,
string para2, DataSet*& dset_para2, DataType dtype_para2)
{
try {
Exception::dontPrint(); //to handle errors
@ -351,6 +392,11 @@ public:
}else
dset = new DataSet (fd->createDataSet(dsetname.c_str(), dtype, *dspace));
//create parameter datasets
hsize_t dims[1] = {nDimx};
dspace_para = new DataSpace (1,dims);
dset_para1 = new DataSet(fd->createDataSet(para1.c_str(), dtype_para1, *dspace_para));
dset_para2 = new DataSet(fd->createDataSet(para2.c_str(), dtype_para2, *dspace_para));
}
catch(Exception error){
cprintf(RED,"Error in creating HDF5 handles in object %d\n",ind);
@ -366,13 +412,19 @@ public:
/**
* Create virtual file
* (in C because H5Pset_virtual doesnt exist yet in C++)
* @param fd virtual file handle
* @param virtualfname virtual file name
* @param virtualDatasetname virtual dataset name
* @param srcDatasetname source dataset name
* @param spara1Datasetname source dataset name of parameter 1
* @param vpara1DatasetName virtual dataset name of parameter 1
* @param dtype_para1 datatype of parameter 1
* @param spara1Datasetname source dataset name of parameter 1
* @param vpara2DatasetName virtual dataset name of parameter 2
* @param dtype_para2 datatype of parameter 2
* @param numFiles number of files
* @param fileNames array of file names
* @param owenable overwrite enable
* @param fnum current frame number
* @param dtype datatype
* @param srcNDimx source number of objects in x dimension (Number of images)
* @param srcNDimy source number of objects in y dimension (Number of pixels in y dir)
@ -384,115 +436,222 @@ public:
* @returns 0 for success and 1 for fail
*/
static int CreateVirtualDataFile(hid_t& fd, string virtualfname, string virtualDatasetname, string srcDatasetname,
int numFiles, string fileNames[], bool owenable, uint64_t fnum, hid_t dtype,
string spara1DatasetName, string vpara1DatasetName, hid_t dtype_para1,
string spara2DatasetName, string vpara2DatasetName, hid_t dtype_para2,
int numFiles, string fileNames[], bool owenable, hid_t dtype,
uint64_t srcNDimx, uint32_t srcNDimy, uint32_t srcNDimz,
uint64_t dstNDimx, uint32_t dstNDimy, uint32_t dstNDimz, double version)
{
//file
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal >= 0) {
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) >= 0) {
if(!owenable) fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal);
else fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (fd >= 0) {
if (dfal < 0)
return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualfname + string("\n"));
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualfname + string("\n"));
if(!owenable)
fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal);
else
fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (fd < 0)
return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualfname + string("\n"));
//attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr >= 0) {
hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid >= 0) {
double attr_data = version;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) >= 0) {
if (H5Aclose (attrid) >= 0) {
//dataspace
hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace >= 0) {
hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
if (srcDataspace >= 0) {
//attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr < 0)
return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualfname + string("\n"));
hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid < 0)
return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualfname + string("\n"));
double attr_data = version;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualfname + string("\n"));
if (H5Aclose (attrid) < 0)
return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualfname + string("\n"));
//fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl >= 0) {
int fill_value = -1;
if (H5Pset_fill_value (dcpl, dtype, &fill_value) >= 0) {
//hyperslab
hsize_t offset[3]={0,0,0},count[3]={srcNDimx,srcNDimy, srcNDimz};
bool error = false;
for (int i = 0; i < numFiles; i++) {
//cout<<"("<<offset[0]<<","<<offset[1]<<","<<offset[2]<<")"<<endl;
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
cprintf(RED,"could not select hyperslab\n");
error = true;
break;
}
if (H5Pset_virtual(dcpl, vdsDataspace, fileNames[i].c_str(), srcDatasetname.c_str(), srcDataspace) < 0) {
cprintf(RED,"could not set mapping\n");
error = true;
break;
}
offset[2] += srcNDimz;
if(offset[2] >= dstNDimz){
offset[2] = 0;
offset[1] += srcNDimy;
}
}
if (!error) {
//dataset
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset >= 0){
//**data dataset**
//dataspace
hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace < 0)
return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualfname + string("\n"));
hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
if (srcDataspace < 0)
return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualfname + string("\n"));
H5Sclose(vdsDataspace);vdsDataspace = 0;
H5Sclose(srcDataspace);srcDataspace = 0;
H5Dclose(vdsdataset);vdsdataset = 0;
H5Fclose(fd);fd = 0;
return 0;
//fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl < 0)
return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualfname + string("\n"));
int fill_value = -1;
if (H5Pset_fill_value (dcpl, dtype, &fill_value) < 0)
return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualfname + string("\n"));
} else cprintf(RED, "could not create virtual dataset in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not map files in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not fill values in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create dcpl in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create source dataspace in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create virtual dataspace in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not close attribute in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not write attribute in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create attribute in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create dataspace for attribute in virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not set strong file close degree for virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not create dfal for virtual file %s\n", virtualfname.c_str());
H5Fclose(fd);
fd = 0;
return 1;
//hyperslab
hsize_t offset[3]={0, 0, 0},
count[3]={srcNDimx, srcNDimy, srcNDimz};
bool error = false;
for (int i = 0; i < numFiles; i++) {
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
cprintf(RED,"could not select hyperslab\n");
error = true;
break;
}
if (H5Pset_virtual(dcpl, vdsDataspace, fileNames[i].c_str(), srcDatasetname.c_str(), srcDataspace) < 0) {
cprintf(RED,"could not set mapping\n");
error = true;
break;
}
offset[2] += srcNDimz;
if (offset[2] >= dstNDimz) {
offset[2] = 0;
offset[1] += srcNDimy;
}
}
if (error)
return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualfname + string("\n"));
//dataset
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset < 0)
return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualfname + string("\n"));
//close
H5Sclose(vdsDataspace);
H5Sclose(srcDataspace);
H5Dclose(vdsdataset);
//**paramter datasets**
//dataspace
hsize_t vdsdims_para[2] = {dstNDimx, numFiles};
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
if (vdsDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
hsize_t srcdims_para[1] = {srcNDimx};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
if (srcDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
//fill values
hid_t dcpl1 = H5Pcreate (H5P_DATASET_CREATE);
hid_t dcpl2 = H5Pcreate (H5P_DATASET_CREATE);
if ((dcpl1 < 0) || (dcpl2 < 0))
return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualfname + string("\n"));
fill_value = -1;
if ((H5Pset_fill_value (dcpl1, dtype_para1, &fill_value) < 0) || (H5Pset_fill_value (dcpl2, dtype_para2, &fill_value) < 0))
return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualfname + string("\n"));
//hyperslab
hsize_t offset_para[2] = {0,0},
count_para[2] = {dstNDimx,1};
error = false;
for (int i = 0; i < numFiles; i++) {
//cout<<"("<<offset[0]<<","<<offset[1]<<","<<offset[2]<<")"<<endl;
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
cprintf(RED,"could not select hyperslab\n");
error = true;
break;
}
if (H5Pset_virtual(dcpl1, vdsDataspace_para, fileNames[i].c_str(), spara1DatasetName.c_str(), srcDataspace_para) < 0) {
cprintf(RED,"could not set mapping for paramter 1\n");
error = true;
break;
}
if (H5Pset_virtual(dcpl2, vdsDataspace_para, fileNames[i].c_str(), spara2DatasetName.c_str(), srcDataspace_para) < 0) {
cprintf(RED,"could not set mapping for paramter 2\n");
error = true;
break;
}
offset_para[1] ++;
}
if (error)
return CloseFileOnError(fd, string("Error in mapping files (parameters) in virtual file ") + virtualfname + string("\n"));
//dataset
hid_t vdsdataset1 = H5Dcreate2 (fd, vpara1DatasetName.c_str(), dtype_para1, vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT);
hid_t vdsdataset2 = H5Dcreate2 (fd, vpara2DatasetName.c_str(), dtype_para2, vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT);
if ((vdsdataset1 < 0) && (vdsdataset1 < 0))
return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualfname + string("\n"));
//close
H5Sclose(vdsDataspace_para);
H5Sclose(srcDataspace_para);
H5Dclose(vdsdataset1);
H5Dclose(vdsdataset2);
H5Fclose(fd); fd = 0;
return 0;
}
/**
* Copy file to another file (mainly to view virutal files in hdfviewer)
* @param dataType data type
* @param owenable overwrite enable
* @param oldFileName file name including path of file to copy
* @param oldDatasetName dataset name to copy
* @param newFileName file name including path of file to copy to
* @param newDatasetName dataset name to copy to
* @param rank rank
* @param nDimx Number of objects in x dimension
* @param nDimy Number of objects in y dimension
* @param nDimz Number of objects in z dimension
* @param dataType data type
* @returns 0 for success and 1 for fail
*/
template <typename T>
static int CopyVirtualFile(bool owenable, string oldFileName, string oldDatasetName,
string newFileName, string newDatasetName,
uint64_t nDimx, uint32_t nDimy, uint32_t nDimz, T datatype)
static int CopyVirtualFile(T datatype, bool owenable, string oldFileName, string oldDatasetName,
string newFileName, string newDatasetName, int rank,
uint64_t nDimx, uint32_t nDimy, uint32_t nDimz=0)
{
T *data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy*nDimz));
T *data_out;
switch (rank) {
case 2:
data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy));
break;
case 3:
data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy*nDimz));
break;
default:
cprintf(RED,"invalid rank. Options: 2 or 3\n");
return 0;
}
if (datatype == PredType::STD_U16LE) {
printf("datatype:16\n");
} else if (datatype == PredType::STD_U32LE) {
printf("datatype:32\n");
} else if (datatype == PredType::STD_U64LE) {
printf("datatype:64\n");
} else if (datatype == PredType::STD_U8LE) {
printf("datatype:8\n");
} else {
cprintf(RED, "unknown datatype\n");
return 1;
}
printf("owenable:%d\n"
"oldFileName:%s\n"
"oldDatasetName:%s\n"
"newFileName:%s\n"
"newDatasetName:%s\n"
"rank:%d\n"
"nDimx:%llu\n"
"nDimy:%u\n"
"nDimz:%u\n",
owenable?1:0,
oldFileName.c_str(),
oldDatasetName.c_str(),
newFileName.c_str(),
newDatasetName.c_str(),
rank,
(long long unsigned int)nDimx,
nDimy,
nDimz);
H5File* oldfd;
H5File* newfd;
@ -512,8 +671,14 @@ public:
else
newfd = new H5File( newFileName.c_str(), H5F_ACC_TRUNC, NULL, fapl );
//dataspace and dataset
hsize_t dims[3] = {nDimx, nDimy, nDimz};
DataSpace* newDataspace = new DataSpace (3,dims);
DataSpace* newDataspace;
if (rank == 3) {
hsize_t dims[3] = {nDimx, nDimy, nDimz};
newDataspace = new DataSpace (3,dims);
} else if (rank == 2) {
hsize_t dims[2] = {nDimx, nDimy};
newDataspace = new DataSpace (2,dims);
}
DataSet* newDataset = new DataSet( newfd->createDataSet(newDatasetName.c_str(), datatype, *newDataspace));
//write and close
newDataset->write(data_out,datatype);
@ -537,46 +702,90 @@ public:
/**
* Link the Virtual File in the Master File
* In C because H5Lcreate_external exists only in C
* @param masterFileName master file name
* @param virtualfname virtual file name
* @param virtualDatasetname virtual dataset name
* @param masterFileName master file name
* @returns 0 for success and 1 for fail
*/
int LinkVirtualInMaster(string virtualfname, string virtualDatasetname, string masterFileName) {
int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname,
string virtualpara1DatasetName, string virtualpara2DatasetName) {
char linkname[100];
sprintf(linkname, "/entry/data/%s",virtualDatasetname.c_str());
hid_t mfd=-1, vfd=-1, vdset=-1;
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) >= 0) {
mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal);
if (mfd >= 0) {
vfd = H5Fopen( virtualfname.c_str(), H5F_ACC_RDWR, dfal);
if (vfd >= 0) {
vdset = H5Dopen2( vfd, virtualDatasetname.c_str(), H5P_DEFAULT);
if (vdset >= 0) {
if(H5Lcreate_external( virtualfname.c_str(), virtualDatasetname.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) >= 0) {
H5Dclose(vdset);
H5Fclose(mfd);
H5Fclose(vfd);
return 0;
cprintf(BLUE,"link created!\n");
} else cprintf(RED, "could not create link\n");
} else cprintf(RED, "could not open virtual dataset %s\n", virtualDatasetname.c_str());
} else cprintf(RED, "could not open virtual file %s\n", virtualfname.c_str());
} else cprintf(RED, "could not open master file %s\n", masterFileName.c_str());
}else cprintf(RED, "could not set strong file close degree\n");
hid_t vfd = 0;
if(vdset >=0 ) H5Dclose(vdset);
if(mfd >=0 ) H5Fclose(mfd);
if(vfd >=0 ) H5Fclose(vfd);
return 1;
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
return CloseFileOnError( vfd, string("Error in creating file access property for link\n"));
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
return CloseFileOnError( vfd, string("Error in setting strong file close degree for link\n"));
//open master file
hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal);
if (mfd < 0)
return CloseFileOnError( vfd, string("Error in opening master file\n"));
//open virtual file
vfd = H5Fopen( virtualfname.c_str(), H5F_ACC_RDWR, dfal);
if (vfd < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in opening virtual file\n"));
}
//**data dataset**
hid_t vdset = H5Dopen2( vfd, virtualDatasetname.c_str(), H5P_DEFAULT);
if (vdset < 0) {
H5Fclose(mfd);
return CloseFileOnError( vfd, string("Error in opening virtual data dataset\n"));
}
sprintf(linkname, "/entry/data/%s",virtualDatasetname.c_str());
if(H5Lcreate_external( virtualfname.c_str(), virtualDatasetname.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in creating link to data dataset\n"));
}
H5Dclose(vdset);
//**paramter datasets**
hid_t vdset1 = H5Dopen2( vfd, virtualpara1DatasetName.c_str(), H5P_DEFAULT);
hid_t vdset2 = H5Dopen2( vfd, virtualpara2DatasetName.c_str(), H5P_DEFAULT);
if ((vdset1 < 0) || (vdset2 < 0)) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in opening virtual parameter dataet to create link\n"));
}
sprintf(linkname, "/entry/data/%s",virtualpara1DatasetName.c_str());
if(H5Lcreate_external( virtualfname.c_str(), virtualpara1DatasetName.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in creating link to virtual parameter 1 dataset\n"));
}
sprintf(linkname, "/entry/data/%s",virtualpara2DatasetName.c_str());
if(H5Lcreate_external( virtualfname.c_str(), virtualpara2DatasetName.c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in creating link to virtual parameter 2 dataset\n"));
}
H5Dclose(vdset1);
H5Dclose(vdset2);
H5Fclose(mfd); mfd = 0;
H5Fclose(vfd); vfd = 0;
return 0;
}
/**
* Print Error msg and Close File and called on error
* @returns 1 for fail
*/
static int CloseFileOnError(hid_t& fd, const string msg) {
cprintf(RED, "%s", msg.c_str());
if(fd > 0) H5Fclose(fd); fd = 0;
return 1;
}
};
#endif

6
slsReceiverSoftware/include/Listener.h
View File

@ -27,8 +27,9 @@ class Listener : private virtual slsReceiverDefs, public ThreadObject {
* @param e ethernet interface
* @param act pointer to activated
* @param nf pointer to number of images to catch
* @param dr pointer to dynamic range
*/
Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf);
Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf, uint32_t* dr);
/**
* Destructor
@ -236,6 +237,9 @@ class Listener : private virtual slsReceiverDefs, public ThreadObject {
/** Number of Images to catch */
uint64_t* numImages;
/** Dynamic Range */
uint32_t* dynamicRange;
/**Number of complete Packets caught for an entire acquisition (including all scans) */
uint64_t numTotalPacketsCaught;

9
slsReceiverSoftware/src/DataStreamer.cpp
View File

@ -63,6 +63,7 @@ DataStreamer::DataStreamer(Fifo*& f, uint32_t* dr, uint32_t* freq, uint32_t* tim
memset((void*)&timerBegin, 0, sizeof(timespec));
currentHeader = new char[255];
strcpy(fileNametoStream, "");
}
@ -121,9 +122,11 @@ void DataStreamer::ResetParametersforNewAcquisition() {
acquisitionStartedFlag = false;
}
void DataStreamer::ResetParametersforNewMeasurement(){
void DataStreamer::ResetParametersforNewMeasurement(char* fname){
firstMeasurementIndex = 0;
measurementStartedFlag = false;
strcpy(fileNametoStream, fname);
cprintf(BLUE,"fname:%s\n",fname);
CreateHeaderPart1();
}
@ -312,17 +315,15 @@ int DataStreamer::SendHeader(uint64_t fnum, uint32_t snum, bool dummy) {
uint64_t frameIndex = -1;
uint64_t acquisitionIndex = -1;
uint32_t subframeIndex = -1;
char fname[MAX_STR_LENGTH] = "run";
char buf[1000] = "";
if (!dummy) {
frameIndex = fnum - firstMeasurementIndex;
acquisitionIndex = fnum - firstAcquisitionIndex;
subframeIndex = snum;
/* fname to be included in fifo buffer? */
}
int len = sprintf(buf, jsonHeaderFormat, currentHeader, acquisitionIndex, frameIndex, subframeIndex, fname);
int len = sprintf(buf, jsonHeaderFormat, currentHeader, acquisitionIndex, frameIndex, subframeIndex, fileNametoStream);
#ifdef VERBOSE
printf("%d Streamer: buf:%s\n", index, buf);
#endif

56
slsReceiverSoftware/src/HDF5File.cpp
View File

@ -26,6 +26,17 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
dataspace(0),
dataset(0),
datatype(PredType::STD_U16LE),
dataspace_para(0),
para1("sub_frame_number"),
dataset_para1(0),
datatype_para1(PredType::STD_U32LE),
para2("bunch_id"),
dataset_para2(0),
datatype_para2(PredType::STD_U64LE),
nPixelsX(nx),
nPixelsY(ny),
numFramesInFile(0)
@ -86,12 +97,14 @@ int HDF5File::CreateFile(uint64_t fnum) {
if(!fnum) UpdateDataType();
uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum);
pthread_mutex_lock(&Mutex);
if (HDF5FileStatic::CreateDataFile(index, *overWriteEnable, currentFileName, *frameIndexEnable,
fnum, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
datatype, filefd, dataspace, dataset,
HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES) == FAIL) {
HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES,
dataspace_para,
para1, dataset_para1, datatype_para1,
para2, dataset_para2, datatype_para2) == FAIL) {
pthread_mutex_unlock(&Mutex);
return FAIL;
}
@ -107,7 +120,7 @@ int HDF5File::CreateFile(uint64_t fnum) {
void HDF5File::CloseCurrentFile() {
pthread_mutex_lock(&Mutex);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
if (master && (*detIndex==0)) {
HDF5FileStatic::CloseVirtualDataFile(virtualfd);
}
@ -116,7 +129,7 @@ void HDF5File::CloseCurrentFile() {
void HDF5File::CloseAllFiles() {
pthread_mutex_lock(&Mutex);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
if (master && (*detIndex==0)) {
HDF5FileStatic::CloseMasterDataFile(masterfd);
HDF5FileStatic::CloseVirtualDataFile(virtualfd);
@ -126,17 +139,25 @@ void HDF5File::CloseAllFiles() {
int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
if (numFramesInFile >= maxFramesPerFile) {/**max *100?????????????*/
if (numFramesInFile >= maxFramesPerFile) {/**max multiply by 100?????????????*/
CloseCurrentFile();
CreateFile(fnum);
}
numFramesInFile++;
uint32_t snum = (*((uint32_t*)(buffer + FILE_FRAME_HDR_FNUM_SIZE)));
uint64_t bid = (*((uint64_t*)(buffer + FILE_FRAME_HDR_FNUM_SIZE + FILE_FRAME_HDR_SNUM_SIZE)));
pthread_mutex_lock(&Mutex);
if (HDF5FileStatic::WriteDataFile(index, buffer + FILE_FRAME_HEADER_SIZE, /** ignoring bunchid?????????? */
if (HDF5FileStatic::WriteDataFile(index, buffer + FILE_FRAME_HEADER_SIZE,
fnum%maxFramesPerFile, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
dataspace, dataset, datatype) == OK) {
pthread_mutex_unlock(&Mutex);
return OK;
if (HDF5FileStatic::WriteParameterDatasets(index, dataspace_para,
fnum%maxFramesPerFile,
dataset_para1, datatype_para1, &snum,
dataset_para2, datatype_para2, &bid) == OK) {
pthread_mutex_unlock(&Mutex);
return OK;
}
}
pthread_mutex_unlock(&Mutex);
cprintf(RED,"%d Error: Write to file failed\n", index);
@ -202,18 +223,31 @@ int HDF5File::CreateVirtualFile(uint64_t fnum) {
osfn << "/virtual_data";
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string virtualDatasetName = osfn.str();
//parameter 1 name
osfn.str(""); osfn.clear();
osfn << "/virtual_" << para1;
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string vpara1DatasetName = osfn.str();
//parameter 2 name
osfn.str(""); osfn.clear();
osfn << "/virtual_" << para2;
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string vpara2DatasetName = osfn.str();
uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum);
//create virtual file
pthread_mutex_lock(&Mutex);
int ret = HDF5FileStatic::CreateVirtualDataFile(virtualfd, virtualFileName, virtualDatasetName, srcDatasetName,
numReadouts, fileNames, *overWriteEnable, fnum, cdatatype,
para1, vpara1DatasetName, H5T_STD_U32LE,
para2, vpara2DatasetName, H5T_STD_U64LE,
numReadouts, fileNames, *overWriteEnable, cdatatype,
framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),HDF5_WRITER_VERSION);
framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), HDF5_WRITER_VERSION);
if (ret == OK)
ret = HDF5FileStatic::LinkVirtualInMaster(virtualFileName, virtualDatasetName, masterFileName);
ret = HDF5FileStatic::LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, vpara1DatasetName, vpara2DatasetName);
pthread_mutex_unlock(&Mutex);
return ret;

7
slsReceiverSoftware/src/Listener.cpp
View File

@ -29,7 +29,7 @@ pthread_mutex_t Listener::Mutex = PTHREAD_MUTEX_INITIALIZER;
const GeneralData* Listener::generalData(0);
Listener::Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf) :
Listener::Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf, uint32_t* dr) :
ThreadObject(NumberofListeners),
fifo(f),
acquisitionStartedFlag(false),
@ -40,6 +40,7 @@ Listener::Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act,
eth(e),
activated(act),
numImages(nf),
dynamicRange(dr),
numTotalPacketsCaught(0),
numPacketsCaught(0),
firstAcquisitionIndex(0),
@ -302,7 +303,7 @@ uint32_t Listener::ListenToAnImage(char* buf) {
//look for carry over
if (carryOverFlag) {
//check if its the current image packet
generalData->GetHeaderInfo(index, carryOverPacket, fnum, pnum, snum, bid);
generalData->GetHeaderInfo(index, carryOverPacket, *dynamicRange, fnum, pnum, snum, bid);
if (fnum != currentFrameIndex) {
return generalData->imageSize;
}
@ -327,7 +328,7 @@ uint32_t Listener::ListenToAnImage(char* buf) {
//update parameters
numPacketsCaught++; //record immediately to get more time before socket shutdown
numTotalPacketsCaught++;
generalData->GetHeaderInfo(index, listeningPacket, fnum, pnum, snum, bid);
generalData->GetHeaderInfo(index, listeningPacket, *dynamicRange, fnum, pnum, snum, bid);
lastCaughtFrameIndex = fnum;
#ifdef VERBOSE
if (!index && !pnum) cprintf(GREEN,"Listening %d: fnum:%lld, pnum:%d\n", index, (long long int)fnum, pnum);

11
slsReceiverSoftware/src/UDPStandardImplementation.cpp
View File

@ -393,7 +393,7 @@ int UDPStandardImplementation::setDetectorType(const detectorType d) {
//create threads
for ( int i=0; i < numThreads; ++i ) {
listener.push_back(new Listener(fifo[i], &status, &udpPortNum[i], eth, &activated, &numberOfFrames));
listener.push_back(new Listener(fifo[i], &status, &udpPortNum[i], eth, &activated, &numberOfFrames, &dynamicRange));
dataProcessor.push_back(new DataProcessor(fifo[i], &fileFormatType, &fileWriteEnable, &dataStreamEnable,
&callbackAction, rawDataReadyCallBack,pRawDataReady));
if (Listener::GetErrorMask() || DataProcessor::GetErrorMask()) {
@ -780,8 +780,13 @@ void UDPStandardImplementation::ResetParametersforNewMeasurement() {
(*it)->ResetParametersforNewMeasurement();
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it)
(*it)->ResetParametersforNewMeasurement();
for (vector<DataStreamer*>::const_iterator it = dataStreamer.begin(); it != dataStreamer.end(); ++it)
(*it)->ResetParametersforNewMeasurement();
if (dataStreamEnable) {
char fnametostream[1000];
sprintf(fnametostream, "%s/%s", filePath, fileName);
for (vector<DataStreamer*>::const_iterator it = dataStreamer.begin(); it != dataStreamer.end(); ++it)
(*it)->ResetParametersforNewMeasurement(fnametostream);
}
}