detectors/slsDetectorPackage
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/slsDetectorPackage.git synced 2025-06-09 03:20:42 +02:00
Code Issues Actions Packages Releases Activity

changed to have only one virtual file with bunch id, subfnum and data datasets all linked in master file

Browse Source
This commit is contained in:
Dhanya Maliakal 2017-03-03 08:39:53 +01:00
parent 537260879a
commit d8b0a13ee2
7 changed files with 268 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
slsReceiverSoftware/include/DataProcessor.h
View File

@ -174,6 +174,11 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
*/ */
void CloseFiles(); void CloseFiles();
/**
* End of Acquisition
* @param numf number of images caught
*/
void EndofAcquisition(uint64_t numf);
private: private:

8
slsReceiverSoftware/include/File.h
View File

@ -141,7 +141,13 @@ class File : private virtual slsReceiverDefs {
cprintf(RED,"This is a generic function SetNumberofPixels that should be overloaded by a derived class\n"); cprintf(RED,"This is a generic function SetNumberofPixels that should be overloaded by a derived class\n");
} }
/**
* End of Acquisition
* @param numf number of images caught
*/
virtual void EndofAcquisition(uint64_t numf) {
cprintf(RED,"This is a generic function EndofAcquisition that should be overloaded by a derived class\n");
}
protected: protected:

31
slsReceiverSoftware/include/HDF5File.h
View File

@ -101,10 +101,17 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap); uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap);
/** /**
* Create Virtual File * End of Acquisition
* @param fnum frame number * @param numf number of images caught
*/ */
int CreateVirtualFile(uint64_t fnum); void EndofAcquisition(uint64_t numf);
/**
* Create Virtual File
* @param numf number of images caught
* @returns OK or FAIL
*/
int CreateVirtualFile(uint64_t numf);
private: private:
@ -122,7 +129,6 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
/** mutex to update static items among objects (threads)*/ /** mutex to update static items among objects (threads)*/
static pthread_mutex_t Mutex; static pthread_mutex_t Mutex;
@ -144,6 +150,16 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
/** Datatype of dataset */ /** Datatype of dataset */
DataType datatype; DataType datatype;
/** Number of pixels in x direction */
uint32_t nPixelsX;
/** Number of pixels in y direction */
uint32_t nPixelsY;
/** Number of frames in file */
uint32_t numFramesInFile;
//parameters
/** Dataspace of parameters */ /** Dataspace of parameters */
DataSpace* dataspace_para; DataSpace* dataspace_para;
@ -165,14 +181,7 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
/** Datatype of parameter2 */ /** Datatype of parameter2 */
DataType datatype_para2; DataType datatype_para2;
/** Number of pixels in x direction */
uint32_t nPixelsX;
/** Number of pixels in y direction */
uint32_t nPixelsY;
/** Number of frames in file */
uint32_t numFramesInFile;
}; };
#endif #endif

279
slsReceiverSoftware/include/HDF5FileStatic.h
View File

@ -80,17 +80,13 @@ public:
* @param fnameprefix file name prefix (includes scan and position variables) * @param fnameprefix file name prefix (includes scan and position variables)
* @param fnum current frame number * @param fnum current frame number
* @param findex file index * @param findex file index
* @param frindexenable frame index enable
* @param fnum frame number index
* @returns virtual file name * @returns virtual file name
*/ */
static string CreateVirtualFileName(char* fpath, char* fnameprefix, uint64_t findex, bool frindexenable, static string CreateVirtualFileName(char* fpath, char* fnameprefix, uint64_t findex)
uint64_t fnum)
{ {
ostringstream osfn; ostringstream osfn;
osfn << fpath << "/" << fnameprefix; osfn << fpath << "/" << fnameprefix;
osfn << "_virtual"; osfn << "_virtual";
if (frindexenable) osfn << "_f" << setfill('0') << setw(12) << fnum;
osfn << "_" << findex; osfn << "_" << findex;
osfn << ".h5"; osfn << ".h5";
return osfn.str(); return osfn.str();
@ -408,185 +404,198 @@ public:
} }
/** /**
* Create virtual file * Create virtual file
* (in C because H5Pset_virtual doesnt exist yet in C++) * (in C because H5Pset_virtual doesnt exist yet in C++)
* @param fd virtual file handle * @param fd virtual file handle
* @param virtualfname virtual file name * @param masterFileName master file name
* @param virtualDatasetname virtual dataset name * @param fpath file path
* @param srcDatasetname source dataset name * @param fnameprefix file name prefix (includes scan and position variables)
* @param spara1Datasetname source dataset name of parameter 1 * @param findex file index
* @param vpara1DatasetName virtual dataset name of parameter 1 * @param frindexenable frame index enable
* @param dtype_para1 datatype of parameter 1 * @param dindex readout index
* @param spara1Datasetname source dataset name of parameter 1 * @param numunits number of units per readout. eg. eiger has 2 udp units per readout
* @param vpara2DatasetName virtual dataset name of parameter 2 * @param maxFramesPerFile maximum frames per file
* @param dtype_para2 datatype of parameter 2 * @param numf number of frames caught
* @param numFiles number of files * @param srcDataseName source dataset name
* @param fileNames array of file names * @param srcP1DatasetName source parameter 1 dataset name
* @param owenable overwrite enable * @param srcP2DatasetName source parameter 2 dataset name
* @param dtype datatype * @param dataType datatype of data dataset
* @param srcNDimx source number of objects in x dimension (Number of images) * @param p1DataType datatype of parameter 1
* @param srcNDimy source number of objects in y dimension (Number of pixels in y dir) * @param p2DataType datatype of parameter 2
* @param srcNDimz source number of objects in z dimension (Number of pixels in x dir) * @param numDety number of readouts in Y dir
* @param dstNDimx destination number of objects in x dimension (Number of images) * @param numDetz number of readouts in Z dir
* @param dstNDimy destination number of objects in y dimension (Number of pixels in y dir) * @param nDimy number of objects in y dimension in source file (Number of pixels in y dir)
* @param dstNDimz destination number of objects in z dimension (Number of pixels in x dir) * @param nDimz number of objects in z dimension in source file (Number of pixels in x dir)
* @param version version of software for hdf5 writing * @param version version of software for hdf5 writing
* @returns 0 for success and 1 for fail * @returns 0 for success and 1 for fail
*/ */
static int CreateVirtualDataFile(hid_t& fd, string virtualfname, string virtualDatasetname, string srcDatasetname, static int CreateVirtualDataFile(
string spara1DatasetName, string vpara1DatasetName, hid_t dtype_para1, hid_t& fd, string masterFileName,
string spara2DatasetName, string vpara2DatasetName, hid_t dtype_para2, char* fpath, char* fnameprefix, uint64_t findex, bool frindexenable,
int numFiles, string fileNames[], bool owenable, hid_t dtype, int dindex, int numunits,
uint64_t srcNDimx, uint32_t srcNDimy, uint32_t srcNDimz, uint32_t maxFramesPerFile, uint64_t numf,
uint64_t dstNDimx, uint32_t dstNDimy, uint32_t dstNDimz, double version) string srcDataseName, string srcP1DatasetName, string srcP2DatasetName,
DataType dataType, DataType p1DataType, DataType p2DataType,
int numDety, int numDetz, uint32_t nDimy, uint32_t nDimz,
double version)
{ {
//virtual names
string virtualFileName = CreateVirtualFileName(fpath, fnameprefix, findex);
printf("Virtual File: %s\n", virtualFileName.c_str());
//file //file
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0) if (dfal < 0)
return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualFileName + string("\n"));
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualFileName + string("\n"));
if(!owenable) fd = H5Fcreate( virtualFileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal);
else
fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (fd < 0) if (fd < 0)
return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualFileName + string("\n"));
//attributes - version //attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR); hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr < 0) if (dataspace_attr < 0)
return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualFileName + string("\n"));
hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT); hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid < 0) if (attrid < 0)
return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualFileName + string("\n"));
double attr_data = version; double attr_data = version;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0) if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualFileName + string("\n"));
if (H5Aclose (attrid) < 0) if (H5Aclose (attrid) < 0)
return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualFileName + string("\n"));
//virtual dataspace
//**data dataset** hsize_t vdsdims[3] = {numf, numDety * nDimy, numDetz * nDimz};
//dataspace
hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL); hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace < 0) if (vdsDataspace < 0)
return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualFileName + string("\n"));
hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz}; hsize_t vdsdims_para[2] = {numf, numDety * numDetz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL); hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
if (srcDataspace < 0) if (vdsDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
//fill values //fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE); hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl < 0) if (dcpl < 0)
return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualFileName + string("\n"));
int fill_value = -1; int fill_value = -1;
if (H5Pset_fill_value (dcpl, dtype, &fill_value) < 0) if (H5Pset_fill_value (dcpl, GetDataTypeinC(dataType), &fill_value) < 0)
return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualFileName + string("\n"));
hid_t dcpl1 = H5Pcreate (H5P_DATASET_CREATE);
hid_t dcpl2 = H5Pcreate (H5P_DATASET_CREATE);
if ((dcpl1 < 0) || (dcpl2 < 0))
return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualFileName + string("\n"));
fill_value = -1;
if ((H5Pset_fill_value (dcpl1, GetDataTypeinC(p1DataType), &fill_value) < 0) ||
(H5Pset_fill_value (dcpl2, GetDataTypeinC(p2DataType), &fill_value) < 0))
return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualFileName + string("\n"));
//hyperslab //hyperslab
hsize_t offset[3]={0, 0, 0}, int numMajorHyperslab = numf/maxFramesPerFile;
count[3]={srcNDimx, srcNDimy, srcNDimz}; if (numf%maxFramesPerFile) numMajorHyperslab++;
bool error = false; bool error = false;
for (int i = 0; i < numFiles; i++) { uint64_t framesSaved = 0;
for (int j = 0; j < numMajorHyperslab; j++) {
uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numf-framesSaved);
hsize_t offset[3] = {framesSaved, 0, 0};
hsize_t count[3] = {nDimx, nDimy, nDimz};
hsize_t offset_para[2] = {framesSaved, 0};
hsize_t count_para[2] = {nDimx, 1};
for (int i = 0; i < numDety * numDetz; ++i) {
//setect hyperslabs
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) { if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
cprintf(RED,"could not select hyperslab\n"); cprintf(RED,"could not select hyperslab\n");
error = true; error = true;
break; break;
} }
if (H5Pset_virtual(dcpl, vdsDataspace, fileNames[i].c_str(), srcDatasetname.c_str(), srcDataspace) < 0) { if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
cprintf(RED,"could not set mapping\n"); cprintf(RED,"could not select hyperslab for parameters\n");
error = true; error = true;
break; break;
} }
offset[2] += srcNDimz;
if (offset[2] >= dstNDimz) {
offset[2] = 0;
offset[1] += srcNDimy;
}
}
if (error)
return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualfname + string("\n"));
//dataset //source file name
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); string srcFileName = HDF5FileStatic::CreateFileName(fpath, fnameprefix, findex,
if (vdsdataset < 0) frindexenable, framesSaved, dindex, numunits, i);
return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualfname + string("\n"));
//close //source dataset name
H5Sclose(vdsDataspace); ostringstream osfn;
H5Sclose(srcDataspace); osfn << "/data";
H5Dclose(vdsdataset); if (frindexenable) osfn << "_f" << setfill('0') << setw(12) << framesSaved;
string srcDatasetName = osfn.str();
//source dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
//**paramter datasets** hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
//dataspace if (srcDataspace < 0)
hsize_t vdsdims_para[2] = {dstNDimx, numFiles}; return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualFileName + string("\n"));
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL); hsize_t srcdims_para[1] = {nDimx};
if (vdsDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
hsize_t srcdims_para[1] = {srcNDimx};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL); hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
if (srcDataspace_para < 0) if (srcDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
//fill values //mapping
hid_t dcpl1 = H5Pcreate (H5P_DATASET_CREATE); if (H5Pset_virtual(dcpl, vdsDataspace, srcFileName.c_str(), srcDatasetName.c_str(), srcDataspace) < 0) {
hid_t dcpl2 = H5Pcreate (H5P_DATASET_CREATE);
if ((dcpl1 < 0) || (dcpl2 < 0))
return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualfname + string("\n"));
fill_value = -1;
if ((H5Pset_fill_value (dcpl1, dtype_para1, &fill_value) < 0) || (H5Pset_fill_value (dcpl2, dtype_para2, &fill_value) < 0))
return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualfname + string("\n"));
//hyperslab
hsize_t offset_para[2] = {0,0},
count_para[2] = {dstNDimx,1};
error = false;
for (int i = 0; i < numFiles; i++) {
//cout<<"("<<offset[0]<<","<<offset[1]<<","<<offset[2]<<")"<<endl;
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
cprintf(RED,"could not select hyperslab\n");
error = true;
break;
}
if (H5Pset_virtual(dcpl1, vdsDataspace_para, fileNames[i].c_str(), spara1DatasetName.c_str(), srcDataspace_para) < 0) {
cprintf(RED,"could not set mapping for paramter 1\n"); cprintf(RED,"could not set mapping for paramter 1\n");
error = true; error = true;
break; break;
} }
if (H5Pset_virtual(dcpl2, vdsDataspace_para, fileNames[i].c_str(), spara2DatasetName.c_str(), srcDataspace_para) < 0) {
if (H5Pset_virtual(dcpl1, vdsDataspace_para, srcFileName.c_str(), srcP1DatasetName.c_str(), srcDataspace_para) < 0) {
cprintf(RED,"could not set mapping for paramter 1\n");
error = true;
break;
}
if (H5Pset_virtual(dcpl2, vdsDataspace_para, srcFileName.c_str(), srcP2DatasetName.c_str(), srcDataspace_para) < 0) {
cprintf(RED,"could not set mapping for paramter 2\n"); cprintf(RED,"could not set mapping for paramter 2\n");
error = true; error = true;
break; break;
} }
offset_para[1] ++; //H5Sclose(srcDataspace);
//H5Sclose(srcDataspace_para);
offset[2] += nDimz;
if (offset[2] >= (numDetz * nDimz)) {
offset[2] = 0;
offset[1] += nDimy;
}
offset_para[1]++;
}
framesSaved += nDimx;
} }
if (error) if (error)
return CloseFileOnError(fd, string("Error in mapping files (parameters) in virtual file ") + virtualfname + string("\n")); return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualFileName + string("\n"));
//dataset //dataset
hid_t vdsdataset1 = H5Dcreate2 (fd, vpara1DatasetName.c_str(), dtype_para1, vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT); string virtualDatasetName = string("/virtual_") + srcDataseName;
hid_t vdsdataset2 = H5Dcreate2 (fd, vpara2DatasetName.c_str(), dtype_para2, vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT); string virutalP1DatasetName = string("/virtual_") + srcP1DatasetName;
if ((vdsdataset1 < 0) && (vdsdataset1 < 0)) string virutalP2DatasetName = string("/virtual_") + srcP2DatasetName;
return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualfname + string("\n")); hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(), GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset < 0)
return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualFileName + string("\n"));
hid_t vdsdataset1 = H5Dcreate2 (fd, virutalP1DatasetName.c_str(), GetDataTypeinC(p1DataType), vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT);
hid_t vdsdataset2 = H5Dcreate2 (fd, virutalP2DatasetName.c_str(), GetDataTypeinC(p2DataType), vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT);
if ((vdsdataset1 < 0) || (vdsdataset2 < 0))
return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualFileName + string("\n"));
//close //close
H5Sclose(vdsDataspace);
H5Sclose(vdsDataspace_para); H5Sclose(vdsDataspace_para);
H5Sclose(srcDataspace_para); H5Dclose(vdsdataset);
H5Dclose(vdsdataset1); H5Dclose(vdsdataset1);
H5Dclose(vdsdataset2); H5Dclose(vdsdataset2);
H5Fclose(fd); fd = 0; H5Fclose(fd); fd = 0;
return 0;
//link
return LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, virutalP1DatasetName, virutalP2DatasetName);
} }
@ -699,15 +708,19 @@ public:
return 0; return 0;
} }
/** /**
* Link the Virtual File in the Master File * Link the Virtual File in the Master File
* In C because H5Lcreate_external exists only in C * In C because H5Lcreate_external exists only in C
* @param masterFileName master file name * @param masterFileName master file name
* @param virtualfname virtual file name * @param virtualfname virtual file name
* @param virtualDatasetname virtual dataset name * @param virtualDatasetname virtual dataset name
* @param virtualpara1DatasetName virtual dataset name of parameter 1
* @param virtualpara2DatasetName virtual dataset name of parameter 1
* @returns 0 for success and 1 for fail * @returns 0 for success and 1 for fail
*/ */
int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname, static int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname,
string virtualpara1DatasetName, string virtualpara2DatasetName) { string virtualpara1DatasetName, string virtualpara2DatasetName) {
char linkname[100]; char linkname[100];
hid_t vfd = 0; hid_t vfd = 0;
@ -746,10 +759,18 @@ public:
//**paramter datasets** //**paramter datasets**
hid_t vdset1 = H5Dopen2( vfd, virtualpara1DatasetName.c_str(), H5P_DEFAULT); hid_t vdset1 = H5Dopen2( vfd, virtualpara1DatasetName.c_str(), H5P_DEFAULT);
if (vdset1 < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in opening virtual parameter 1 dataet to create link\n"));
}
hid_t vdset2 = H5Dopen2( vfd, virtualpara2DatasetName.c_str(), H5P_DEFAULT); hid_t vdset2 = H5Dopen2( vfd, virtualpara2DatasetName.c_str(), H5P_DEFAULT);
if (vdset2 < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in opening virtual parameter 2 dataet to create link\n"));
}
if ((vdset1 < 0) || (vdset2 < 0)) { if ((vdset1 < 0) || (vdset2 < 0)) {
H5Fclose(mfd); mfd = 0; H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in opening virtual parameter dataet to create link\n")); return CloseFileOnError( vfd, string("Error in opening either virtual parameter dataet to create link\n"));
} }
sprintf(linkname, "/entry/data/%s",virtualpara1DatasetName.c_str()); sprintf(linkname, "/entry/data/%s",virtualpara1DatasetName.c_str());
if(H5Lcreate_external( virtualfname.c_str(), virtualpara1DatasetName.c_str(), if(H5Lcreate_external( virtualfname.c_str(), virtualpara1DatasetName.c_str(),
@ -783,6 +804,26 @@ public:
} }
/**
* Get Data type in C
* @param dtype datatype in C++
* @returns datatype in C
*/
static hid_t GetDataTypeinC(DataType dtype) {
if (dtype == PredType::STD_U8LE)
return H5T_STD_U8LE;
else if (dtype == PredType::STD_U16LE)
return H5T_STD_U16LE;
else if (dtype == PredType::STD_U32LE)
return H5T_STD_U32LE;
else if (dtype == PredType::STD_U64LE)
return H5T_STD_U64LE;
else {
cprintf(RED, "Invalid Data type\n");
return H5T_STD_U64LE;
}
}
}; };

8
slsReceiverSoftware/src/DataProcessor.cpp
View File

@ -249,6 +249,12 @@ void DataProcessor::CloseFiles() {
file->CloseAllFiles(); file->CloseAllFiles();
} }
void DataProcessor::EndofAcquisition(uint64_t numf) {
if (*fileWriteEnable && file->GetFileType() == HDF5 && numf) {
file->EndofAcquisition(numf);
}
}
void DataProcessor::ThreadExecution() { void DataProcessor::ThreadExecution() {
char* buffer=0; char* buffer=0;
@ -307,7 +313,7 @@ void DataProcessor::ProcessAnImage(char* buf) {
RecordFirstIndices(fnum); RecordFirstIndices(fnum);
} }
if (fileWriteEnable && *callbackAction == DO_EVERYTHING) if (*fileWriteEnable && *callbackAction == DO_EVERYTHING)
file->WriteToFile(buf, generalData->fifoBufferSize + FILE_FRAME_HEADER_SIZE, fnum-firstMeasurementIndex); file->WriteToFile(buf, generalData->fifoBufferSize + FILE_FRAME_HEADER_SIZE, fnum-firstMeasurementIndex);
} }

122
slsReceiverSoftware/src/HDF5File.cpp
View File

@ -26,6 +26,9 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
dataspace(0), dataspace(0),
dataset(0), dataset(0),
datatype(PredType::STD_U16LE), datatype(PredType::STD_U16LE),
nPixelsX(nx),
nPixelsY(ny),
numFramesInFile(0),
dataspace_para(0), dataspace_para(0),
@ -35,11 +38,7 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
para2("bunch_id"), para2("bunch_id"),
dataset_para2(0), dataset_para2(0),
datatype_para2(PredType::STD_U64LE), datatype_para2(PredType::STD_U64LE)
nPixelsX(nx),
nPixelsY(ny),
numFramesInFile(0)
{ {
#ifdef VERBOSE #ifdef VERBOSE
PrintMembers(); PrintMembers();
@ -65,29 +64,27 @@ void HDF5File::PrintMembers() {
} }
} }
void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) { void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) {
nPixelsX = nx; nPixelsX = nx;
nPixelsY = ny; nPixelsY = ny;
} }
slsReceiverDefs::fileFormat HDF5File::GetFileType() { slsReceiverDefs::fileFormat HDF5File::GetFileType() {
return HDF5; return HDF5;
} }
void HDF5File::UpdateDataType() { void HDF5File::UpdateDataType() {
switch(*dynamicRange){ switch(*dynamicRange){
case 16: case 16: datatype = PredType::STD_U16LE; break;
datatype = PredType::STD_U16LE; case 32: datatype = PredType::STD_U32LE; break;
break; default: datatype = PredType::STD_U8LE; break;
case 32:
datatype = PredType::STD_U32LE;
break;
default:
datatype = PredType::STD_U8LE;
break;
} }
} }
int HDF5File::CreateFile(uint64_t fnum) { int HDF5File::CreateFile(uint64_t fnum) {
numFramesInFile = 0; numFramesInFile = 0;
currentFileName = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex, currentFileName = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
@ -110,23 +107,17 @@ int HDF5File::CreateFile(uint64_t fnum) {
} }
pthread_mutex_unlock(&Mutex); pthread_mutex_unlock(&Mutex);
printf("%d HDF5 File: %s\n", index, currentFileName.c_str()); printf("%d HDF5 File: %s\n", index, currentFileName.c_str());
//virtual file
if (master && (*detIndex==0))
return CreateVirtualFile(fnum);
return OK; return OK;
} }
void HDF5File::CloseCurrentFile() { void HDF5File::CloseCurrentFile() {
pthread_mutex_lock(&Mutex); pthread_mutex_lock(&Mutex);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2); HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
if (master && (*detIndex==0)) {
HDF5FileStatic::CloseVirtualDataFile(virtualfd);
}
pthread_mutex_unlock(&Mutex); pthread_mutex_unlock(&Mutex);
} }
void HDF5File::CloseAllFiles() { void HDF5File::CloseAllFiles() {
pthread_mutex_lock(&Mutex); pthread_mutex_lock(&Mutex);
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2); HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
@ -139,7 +130,7 @@ void HDF5File::CloseAllFiles() {
int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) { int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
if (numFramesInFile >= maxFramesPerFile) {/**max multiply by 100?????????????*/ if (numFramesInFile >= maxFramesPerFile) {
CloseCurrentFile(); CloseCurrentFile();
CreateFile(fnum); CreateFile(fnum);
} }
@ -168,6 +159,7 @@ int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
int HDF5File::CreateMasterFile(bool en, uint32_t size, int HDF5File::CreateMasterFile(bool en, uint32_t size,
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap) { uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap) {
if (master && (*detIndex==0)) { if (master && (*detIndex==0)) {
virtualfd = 0;
masterFileName = HDF5FileStatic::CreateMasterFileName(filePath, fileNamePrefix, *fileIndex); masterFileName = HDF5FileStatic::CreateMasterFileName(filePath, fileNamePrefix, *fileIndex);
printf("Master File: %s\n", masterFileName.c_str()); printf("Master File: %s\n", masterFileName.c_str());
pthread_mutex_lock(&Mutex); pthread_mutex_lock(&Mutex);
@ -180,77 +172,29 @@ int HDF5File::CreateMasterFile(bool en, uint32_t size,
} }
int HDF5File::CreateVirtualFile(uint64_t fnum) { void HDF5File::EndofAcquisition(uint64_t numf) {
//not created before
if (!virtualfd)
CreateVirtualFile(numf);
}
int HDF5File::CreateVirtualFile(uint64_t numf) {
if (master && (*detIndex==0)) { if (master && (*detIndex==0)) {
//file name
string virtualFileName = HDF5FileStatic::CreateVirtualFileName(filePath, fileNamePrefix, *fileIndex, *frameIndexEnable, fnum);
printf("Virtual File: %s\n", virtualFileName.c_str());
//source file names
int numReadouts = numDetX * numDetY;
string fileNames[numReadouts];
for (int i = 0; i < numReadouts; ++i) {
fileNames[i] = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
*frameIndexEnable, fnum, *detIndex, *numUnitsPerDetector, i);
#ifdef VERBOSE
printf("%d: File Name: %s\n", i, fileNames[i].c_str());
#endif
}
//datatype
hid_t cdatatype;
switch(*dynamicRange){
case 16:
cdatatype = H5T_STD_U16LE;
break;
case 32:
cdatatype = H5T_STD_U32LE;
break;
default:
cdatatype = H5T_STD_U8LE;
break;
}
//source dataset name
ostringstream osfn;
osfn << "/data";
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string srcDatasetName = osfn.str();
//virtual dataset name
osfn.str(""); osfn.clear();
osfn << "/virtual_data";
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string virtualDatasetName = osfn.str();
//parameter 1 name
osfn.str(""); osfn.clear();
osfn << "/virtual_" << para1;
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string vpara1DatasetName = osfn.str();
//parameter 2 name
osfn.str(""); osfn.clear();
osfn << "/virtual_" << para2;
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
string vpara2DatasetName = osfn.str();
uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum);
//create virtual file
pthread_mutex_lock(&Mutex); pthread_mutex_lock(&Mutex);
int ret = HDF5FileStatic::CreateVirtualDataFile(virtualfd, virtualFileName, virtualDatasetName, srcDatasetName, int ret = HDF5FileStatic::CreateVirtualDataFile(
para1, vpara1DatasetName, H5T_STD_U32LE, virtualfd, masterFileName,
para2, vpara2DatasetName, H5T_STD_U64LE, filePath, fileNamePrefix, *fileIndex, *frameIndexEnable,
numReadouts, fileNames, *overWriteEnable, cdatatype, *detIndex, *numUnitsPerDetector,
framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), maxFramesPerFile, numf,
framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), HDF5_WRITER_VERSION); "data", para1, para2,
datatype, datatype_para1, datatype_para2,
if (ret == OK) numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
ret = HDF5FileStatic::LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, vpara1DatasetName, vpara2DatasetName); HDF5_WRITER_VERSION);
pthread_mutex_unlock(&Mutex); pthread_mutex_unlock(&Mutex);
return ret; return ret;
} }
return OK; return OK;
} }

18
slsReceiverSoftware/src/UDPStandardImplementation.cpp
View File

@ -536,6 +536,17 @@ void UDPStandardImplementation::stopReceiver(){
while(DataProcessor::GetRunningMask()){ while(DataProcessor::GetRunningMask()){
usleep(5000); usleep(5000);
} }
//create virtual file
if (fileWriteEnable && fileFormatType == HDF5) {
uint64_t maxFramescaught = 0;
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
}
if (maxFramescaught)
dataProcessor[0]->EndofAcquisition(maxFramescaught);
}
while(DataStreamer::GetRunningMask()){ while(DataStreamer::GetRunningMask()){
usleep(5000); usleep(5000);
} }
@ -637,8 +648,13 @@ void UDPStandardImplementation::shutDownUDPSockets() {
void UDPStandardImplementation::closeFiles() { void UDPStandardImplementation::closeFiles() {
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) uint64_t maxFramescaught = 0;
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
(*it)->CloseFiles(); (*it)->CloseFiles();
maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
}
if (maxFramescaught)
dataProcessor[0]->EndofAcquisition(maxFramescaught);
} }