mirror of
https://github.com/slsdetectorgroup/slsDetectorPackage.git
synced 2025-06-07 10:30:41 +02:00
changed to have only one virtual file with bunch id, subfnum and data datasets all linked in master file
This commit is contained in:
Dhanya Maliakal
parent
537260879a
commit
d8b0a13ee2
@ -174,6 +174,11 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
|
||||
*/
|
||||
void CloseFiles();
|
||||
|
||||
/**
|
||||
* End of Acquisition
|
||||
* @param numf number of images caught
|
||||
*/
|
||||
void EndofAcquisition(uint64_t numf);
|
||||
|
||||
|
||||
private:
|
||||
|
||||
@ -141,7 +141,13 @@ class File : private virtual slsReceiverDefs {
|
||||
cprintf(RED,"This is a generic function SetNumberofPixels that should be overloaded by a derived class\n");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* End of Acquisition
|
||||
* @param numf number of images caught
|
||||
*/
|
||||
virtual void EndofAcquisition(uint64_t numf) {
|
||||
cprintf(RED,"This is a generic function EndofAcquisition that should be overloaded by a derived class\n");
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
|
||||
@ -101,10 +101,17 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
|
||||
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap);
|
||||
|
||||
/**
|
||||
* Create Virtual File
|
||||
* @param fnum frame number
|
||||
* End of Acquisition
|
||||
* @param numf number of images caught
|
||||
*/
|
||||
int CreateVirtualFile(uint64_t fnum);
|
||||
void EndofAcquisition(uint64_t numf);
|
||||
|
||||
/**
|
||||
* Create Virtual File
|
||||
* @param numf number of images caught
|
||||
* @returns OK or FAIL
|
||||
*/
|
||||
int CreateVirtualFile(uint64_t numf);
|
||||
|
||||
|
||||
private:
|
||||
@ -122,7 +129,6 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
|
||||
|
||||
|
||||
|
||||
|
||||
/** mutex to update static items among objects (threads)*/
|
||||
static pthread_mutex_t Mutex;
|
||||
|
||||
@ -144,6 +150,16 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
|
||||
/** Datatype of dataset */
|
||||
DataType datatype;
|
||||
|
||||
/** Number of pixels in x direction */
|
||||
uint32_t nPixelsX;
|
||||
|
||||
/** Number of pixels in y direction */
|
||||
uint32_t nPixelsY;
|
||||
|
||||
/** Number of frames in file */
|
||||
uint32_t numFramesInFile;
|
||||
|
||||
//parameters
|
||||
/** Dataspace of parameters */
|
||||
DataSpace* dataspace_para;
|
||||
|
||||
@ -165,14 +181,7 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
|
||||
/** Datatype of parameter2 */
|
||||
DataType datatype_para2;
|
||||
|
||||
/** Number of pixels in x direction */
|
||||
uint32_t nPixelsX;
|
||||
|
||||
/** Number of pixels in y direction */
|
||||
uint32_t nPixelsY;
|
||||
|
||||
/** Number of frames in file */
|
||||
uint32_t numFramesInFile;
|
||||
|
||||
};
|
||||
#endif
|
||||
|
||||
@ -80,17 +80,13 @@ public:
|
||||
* @param fnameprefix file name prefix (includes scan and position variables)
|
||||
* @param fnum current frame number
|
||||
* @param findex file index
|
||||
* @param frindexenable frame index enable
|
||||
* @param fnum frame number index
|
||||
* @returns virtual file name
|
||||
*/
|
||||
static string CreateVirtualFileName(char* fpath, char* fnameprefix, uint64_t findex, bool frindexenable,
|
||||
uint64_t fnum)
|
||||
static string CreateVirtualFileName(char* fpath, char* fnameprefix, uint64_t findex)
|
||||
{
|
||||
ostringstream osfn;
|
||||
osfn << fpath << "/" << fnameprefix;
|
||||
osfn << "_virtual";
|
||||
if (frindexenable) osfn << "_f" << setfill('0') << setw(12) << fnum;
|
||||
osfn << "_" << findex;
|
||||
osfn << ".h5";
|
||||
return osfn.str();
|
||||
@ -408,185 +404,198 @@ public:
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Create virtual file
|
||||
* (in C because H5Pset_virtual doesnt exist yet in C++)
|
||||
* @param fd virtual file handle
|
||||
* @param virtualfname virtual file name
|
||||
* @param virtualDatasetname virtual dataset name
|
||||
* @param srcDatasetname source dataset name
|
||||
* @param spara1Datasetname source dataset name of parameter 1
|
||||
* @param vpara1DatasetName virtual dataset name of parameter 1
|
||||
* @param dtype_para1 datatype of parameter 1
|
||||
* @param spara1Datasetname source dataset name of parameter 1
|
||||
* @param vpara2DatasetName virtual dataset name of parameter 2
|
||||
* @param dtype_para2 datatype of parameter 2
|
||||
* @param numFiles number of files
|
||||
* @param fileNames array of file names
|
||||
* @param owenable overwrite enable
|
||||
* @param dtype datatype
|
||||
* @param srcNDimx source number of objects in x dimension (Number of images)
|
||||
* @param srcNDimy source number of objects in y dimension (Number of pixels in y dir)
|
||||
* @param srcNDimz source number of objects in z dimension (Number of pixels in x dir)
|
||||
* @param dstNDimx destination number of objects in x dimension (Number of images)
|
||||
* @param dstNDimy destination number of objects in y dimension (Number of pixels in y dir)
|
||||
* @param dstNDimz destination number of objects in z dimension (Number of pixels in x dir)
|
||||
* @param masterFileName master file name
|
||||
* @param fpath file path
|
||||
* @param fnameprefix file name prefix (includes scan and position variables)
|
||||
* @param findex file index
|
||||
* @param frindexenable frame index enable
|
||||
* @param dindex readout index
|
||||
* @param numunits number of units per readout. eg. eiger has 2 udp units per readout
|
||||
* @param maxFramesPerFile maximum frames per file
|
||||
* @param numf number of frames caught
|
||||
* @param srcDataseName source dataset name
|
||||
* @param srcP1DatasetName source parameter 1 dataset name
|
||||
* @param srcP2DatasetName source parameter 2 dataset name
|
||||
* @param dataType datatype of data dataset
|
||||
* @param p1DataType datatype of parameter 1
|
||||
* @param p2DataType datatype of parameter 2
|
||||
* @param numDety number of readouts in Y dir
|
||||
* @param numDetz number of readouts in Z dir
|
||||
* @param nDimy number of objects in y dimension in source file (Number of pixels in y dir)
|
||||
* @param nDimz number of objects in z dimension in source file (Number of pixels in x dir)
|
||||
* @param version version of software for hdf5 writing
|
||||
* @returns 0 for success and 1 for fail
|
||||
*/
|
||||
static int CreateVirtualDataFile(hid_t& fd, string virtualfname, string virtualDatasetname, string srcDatasetname,
|
||||
string spara1DatasetName, string vpara1DatasetName, hid_t dtype_para1,
|
||||
string spara2DatasetName, string vpara2DatasetName, hid_t dtype_para2,
|
||||
int numFiles, string fileNames[], bool owenable, hid_t dtype,
|
||||
uint64_t srcNDimx, uint32_t srcNDimy, uint32_t srcNDimz,
|
||||
uint64_t dstNDimx, uint32_t dstNDimy, uint32_t dstNDimz, double version)
|
||||
static int CreateVirtualDataFile(
|
||||
hid_t& fd, string masterFileName,
|
||||
char* fpath, char* fnameprefix, uint64_t findex, bool frindexenable,
|
||||
int dindex, int numunits,
|
||||
uint32_t maxFramesPerFile, uint64_t numf,
|
||||
string srcDataseName, string srcP1DatasetName, string srcP2DatasetName,
|
||||
DataType dataType, DataType p1DataType, DataType p2DataType,
|
||||
int numDety, int numDetz, uint32_t nDimy, uint32_t nDimz,
|
||||
double version)
|
||||
{
|
||||
//virtual names
|
||||
string virtualFileName = CreateVirtualFileName(fpath, fnameprefix, findex);
|
||||
printf("Virtual File: %s\n", virtualFileName.c_str());
|
||||
|
||||
//file
|
||||
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
|
||||
if (dfal < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualFileName + string("\n"));
|
||||
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
|
||||
return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualfname + string("\n"));
|
||||
if(!owenable)
|
||||
fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal);
|
||||
else
|
||||
fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
|
||||
return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualFileName + string("\n"));
|
||||
fd = H5Fcreate( virtualFileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
|
||||
if (fd < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualfname + string("\n"));
|
||||
|
||||
return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
//attributes - version
|
||||
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
|
||||
if (dataspace_attr < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualFileName + string("\n"));
|
||||
hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
|
||||
if (attrid < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualFileName + string("\n"));
|
||||
double attr_data = version;
|
||||
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
|
||||
return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualFileName + string("\n"));
|
||||
if (H5Aclose (attrid) < 0)
|
||||
return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
|
||||
|
||||
//**data dataset**
|
||||
//dataspace
|
||||
hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz};
|
||||
//virtual dataspace
|
||||
hsize_t vdsdims[3] = {numf, numDety * nDimy, numDetz * nDimz};
|
||||
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
|
||||
if (vdsDataspace < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualfname + string("\n"));
|
||||
hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz};
|
||||
hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
|
||||
if (srcDataspace < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualFileName + string("\n"));
|
||||
hsize_t vdsdims_para[2] = {numf, numDety * numDetz};
|
||||
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
|
||||
if (vdsDataspace_para < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
|
||||
//fill values
|
||||
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
|
||||
if (dcpl < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualFileName + string("\n"));
|
||||
int fill_value = -1;
|
||||
if (H5Pset_fill_value (dcpl, dtype, &fill_value) < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualfname + string("\n"));
|
||||
|
||||
|
||||
//hyperslab
|
||||
hsize_t offset[3]={0, 0, 0},
|
||||
count[3]={srcNDimx, srcNDimy, srcNDimz};
|
||||
bool error = false;
|
||||
for (int i = 0; i < numFiles; i++) {
|
||||
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
|
||||
cprintf(RED,"could not select hyperslab\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
if (H5Pset_virtual(dcpl, vdsDataspace, fileNames[i].c_str(), srcDatasetname.c_str(), srcDataspace) < 0) {
|
||||
cprintf(RED,"could not set mapping\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
offset[2] += srcNDimz;
|
||||
if (offset[2] >= dstNDimz) {
|
||||
offset[2] = 0;
|
||||
offset[1] += srcNDimy;
|
||||
}
|
||||
}
|
||||
if (error)
|
||||
return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualfname + string("\n"));
|
||||
|
||||
//dataset
|
||||
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
|
||||
if (vdsdataset < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualfname + string("\n"));
|
||||
|
||||
//close
|
||||
H5Sclose(vdsDataspace);
|
||||
H5Sclose(srcDataspace);
|
||||
H5Dclose(vdsdataset);
|
||||
|
||||
|
||||
|
||||
//**paramter datasets**
|
||||
//dataspace
|
||||
hsize_t vdsdims_para[2] = {dstNDimx, numFiles};
|
||||
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
|
||||
if (vdsDataspace_para < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
|
||||
hsize_t srcdims_para[1] = {srcNDimx};
|
||||
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
|
||||
if (srcDataspace_para < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
|
||||
|
||||
//fill values
|
||||
if (H5Pset_fill_value (dcpl, GetDataTypeinC(dataType), &fill_value) < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualFileName + string("\n"));
|
||||
hid_t dcpl1 = H5Pcreate (H5P_DATASET_CREATE);
|
||||
hid_t dcpl2 = H5Pcreate (H5P_DATASET_CREATE);
|
||||
if ((dcpl1 < 0) || (dcpl2 < 0))
|
||||
return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualFileName + string("\n"));
|
||||
fill_value = -1;
|
||||
if ((H5Pset_fill_value (dcpl1, dtype_para1, &fill_value) < 0) || (H5Pset_fill_value (dcpl2, dtype_para2, &fill_value) < 0))
|
||||
return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualfname + string("\n"));
|
||||
if ((H5Pset_fill_value (dcpl1, GetDataTypeinC(p1DataType), &fill_value) < 0) ||
|
||||
(H5Pset_fill_value (dcpl2, GetDataTypeinC(p2DataType), &fill_value) < 0))
|
||||
return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
|
||||
//hyperslab
|
||||
hsize_t offset_para[2] = {0,0},
|
||||
count_para[2] = {dstNDimx,1};
|
||||
error = false;
|
||||
for (int i = 0; i < numFiles; i++) {
|
||||
//cout<<"("<<offset[0]<<","<<offset[1]<<","<<offset[2]<<")"<<endl;
|
||||
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
|
||||
cprintf(RED,"could not select hyperslab\n");
|
||||
error = true;
|
||||
break;
|
||||
int numMajorHyperslab = numf/maxFramesPerFile;
|
||||
if (numf%maxFramesPerFile) numMajorHyperslab++;
|
||||
bool error = false;
|
||||
uint64_t framesSaved = 0;
|
||||
for (int j = 0; j < numMajorHyperslab; j++) {
|
||||
|
||||
uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numf-framesSaved);
|
||||
hsize_t offset[3] = {framesSaved, 0, 0};
|
||||
hsize_t count[3] = {nDimx, nDimy, nDimz};
|
||||
hsize_t offset_para[2] = {framesSaved, 0};
|
||||
hsize_t count_para[2] = {nDimx, 1};
|
||||
|
||||
for (int i = 0; i < numDety * numDetz; ++i) {
|
||||
|
||||
//setect hyperslabs
|
||||
if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
|
||||
cprintf(RED,"could not select hyperslab\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
|
||||
cprintf(RED,"could not select hyperslab for parameters\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
|
||||
//source file name
|
||||
string srcFileName = HDF5FileStatic::CreateFileName(fpath, fnameprefix, findex,
|
||||
frindexenable, framesSaved, dindex, numunits, i);
|
||||
|
||||
//source dataset name
|
||||
ostringstream osfn;
|
||||
osfn << "/data";
|
||||
if (frindexenable) osfn << "_f" << setfill('0') << setw(12) << framesSaved;
|
||||
string srcDatasetName = osfn.str();
|
||||
|
||||
//source dataspace
|
||||
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
|
||||
hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
|
||||
if (srcDataspace < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualFileName + string("\n"));
|
||||
hsize_t srcdims_para[1] = {nDimx};
|
||||
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
|
||||
if (srcDataspace_para < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
//mapping
|
||||
if (H5Pset_virtual(dcpl, vdsDataspace, srcFileName.c_str(), srcDatasetName.c_str(), srcDataspace) < 0) {
|
||||
cprintf(RED,"could not set mapping for paramter 1\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (H5Pset_virtual(dcpl1, vdsDataspace_para, srcFileName.c_str(), srcP1DatasetName.c_str(), srcDataspace_para) < 0) {
|
||||
cprintf(RED,"could not set mapping for paramter 1\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
if (H5Pset_virtual(dcpl2, vdsDataspace_para, srcFileName.c_str(), srcP2DatasetName.c_str(), srcDataspace_para) < 0) {
|
||||
cprintf(RED,"could not set mapping for paramter 2\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
//H5Sclose(srcDataspace);
|
||||
//H5Sclose(srcDataspace_para);
|
||||
offset[2] += nDimz;
|
||||
if (offset[2] >= (numDetz * nDimz)) {
|
||||
offset[2] = 0;
|
||||
offset[1] += nDimy;
|
||||
}
|
||||
offset_para[1]++;
|
||||
}
|
||||
if (H5Pset_virtual(dcpl1, vdsDataspace_para, fileNames[i].c_str(), spara1DatasetName.c_str(), srcDataspace_para) < 0) {
|
||||
cprintf(RED,"could not set mapping for paramter 1\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
if (H5Pset_virtual(dcpl2, vdsDataspace_para, fileNames[i].c_str(), spara2DatasetName.c_str(), srcDataspace_para) < 0) {
|
||||
cprintf(RED,"could not set mapping for paramter 2\n");
|
||||
error = true;
|
||||
break;
|
||||
}
|
||||
offset_para[1] ++;
|
||||
framesSaved += nDimx;
|
||||
}
|
||||
if (error)
|
||||
return CloseFileOnError(fd, string("Error in mapping files (parameters) in virtual file ") + virtualfname + string("\n"));
|
||||
return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
//dataset
|
||||
hid_t vdsdataset1 = H5Dcreate2 (fd, vpara1DatasetName.c_str(), dtype_para1, vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT);
|
||||
hid_t vdsdataset2 = H5Dcreate2 (fd, vpara2DatasetName.c_str(), dtype_para2, vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT);
|
||||
if ((vdsdataset1 < 0) && (vdsdataset1 < 0))
|
||||
return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualfname + string("\n"));
|
||||
string virtualDatasetName = string("/virtual_") + srcDataseName;
|
||||
string virutalP1DatasetName = string("/virtual_") + srcP1DatasetName;
|
||||
string virutalP2DatasetName = string("/virtual_") + srcP2DatasetName;
|
||||
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(), GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
|
||||
if (vdsdataset < 0)
|
||||
return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualFileName + string("\n"));
|
||||
hid_t vdsdataset1 = H5Dcreate2 (fd, virutalP1DatasetName.c_str(), GetDataTypeinC(p1DataType), vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT);
|
||||
hid_t vdsdataset2 = H5Dcreate2 (fd, virutalP2DatasetName.c_str(), GetDataTypeinC(p2DataType), vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT);
|
||||
if ((vdsdataset1 < 0) || (vdsdataset2 < 0))
|
||||
return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualFileName + string("\n"));
|
||||
|
||||
//close
|
||||
H5Sclose(vdsDataspace);
|
||||
H5Sclose(vdsDataspace_para);
|
||||
H5Sclose(srcDataspace_para);
|
||||
H5Dclose(vdsdataset);
|
||||
H5Dclose(vdsdataset1);
|
||||
H5Dclose(vdsdataset2);
|
||||
|
||||
H5Fclose(fd); fd = 0;
|
||||
return 0;
|
||||
|
||||
//link
|
||||
return LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, virutalP1DatasetName, virutalP2DatasetName);
|
||||
}
|
||||
|
||||
|
||||
@ -699,15 +708,19 @@ public:
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Link the Virtual File in the Master File
|
||||
* In C because H5Lcreate_external exists only in C
|
||||
* @param masterFileName master file name
|
||||
* @param virtualfname virtual file name
|
||||
* @param virtualDatasetname virtual dataset name
|
||||
* @param virtualpara1DatasetName virtual dataset name of parameter 1
|
||||
* @param virtualpara2DatasetName virtual dataset name of parameter 1
|
||||
* @returns 0 for success and 1 for fail
|
||||
*/
|
||||
int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname,
|
||||
static int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname,
|
||||
string virtualpara1DatasetName, string virtualpara2DatasetName) {
|
||||
char linkname[100];
|
||||
hid_t vfd = 0;
|
||||
@ -746,10 +759,18 @@ public:
|
||||
|
||||
//**paramter datasets**
|
||||
hid_t vdset1 = H5Dopen2( vfd, virtualpara1DatasetName.c_str(), H5P_DEFAULT);
|
||||
if (vdset1 < 0) {
|
||||
H5Fclose(mfd); mfd = 0;
|
||||
return CloseFileOnError( vfd, string("Error in opening virtual parameter 1 dataet to create link\n"));
|
||||
}
|
||||
hid_t vdset2 = H5Dopen2( vfd, virtualpara2DatasetName.c_str(), H5P_DEFAULT);
|
||||
if (vdset2 < 0) {
|
||||
H5Fclose(mfd); mfd = 0;
|
||||
return CloseFileOnError( vfd, string("Error in opening virtual parameter 2 dataet to create link\n"));
|
||||
}
|
||||
if ((vdset1 < 0) || (vdset2 < 0)) {
|
||||
H5Fclose(mfd); mfd = 0;
|
||||
return CloseFileOnError( vfd, string("Error in opening virtual parameter dataet to create link\n"));
|
||||
return CloseFileOnError( vfd, string("Error in opening either virtual parameter dataet to create link\n"));
|
||||
}
|
||||
sprintf(linkname, "/entry/data/%s",virtualpara1DatasetName.c_str());
|
||||
if(H5Lcreate_external( virtualfname.c_str(), virtualpara1DatasetName.c_str(),
|
||||
@ -783,6 +804,26 @@ public:
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get Data type in C
|
||||
* @param dtype datatype in C++
|
||||
* @returns datatype in C
|
||||
*/
|
||||
static hid_t GetDataTypeinC(DataType dtype) {
|
||||
if (dtype == PredType::STD_U8LE)
|
||||
return H5T_STD_U8LE;
|
||||
else if (dtype == PredType::STD_U16LE)
|
||||
return H5T_STD_U16LE;
|
||||
else if (dtype == PredType::STD_U32LE)
|
||||
return H5T_STD_U32LE;
|
||||
else if (dtype == PredType::STD_U64LE)
|
||||
return H5T_STD_U64LE;
|
||||
else {
|
||||
cprintf(RED, "Invalid Data type\n");
|
||||
return H5T_STD_U64LE;
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -249,6 +249,12 @@ void DataProcessor::CloseFiles() {
|
||||
file->CloseAllFiles();
|
||||
}
|
||||
|
||||
void DataProcessor::EndofAcquisition(uint64_t numf) {
|
||||
if (*fileWriteEnable && file->GetFileType() == HDF5 && numf) {
|
||||
file->EndofAcquisition(numf);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void DataProcessor::ThreadExecution() {
|
||||
char* buffer=0;
|
||||
@ -307,7 +313,7 @@ void DataProcessor::ProcessAnImage(char* buf) {
|
||||
RecordFirstIndices(fnum);
|
||||
}
|
||||
|
||||
if (fileWriteEnable && *callbackAction == DO_EVERYTHING)
|
||||
if (*fileWriteEnable && *callbackAction == DO_EVERYTHING)
|
||||
file->WriteToFile(buf, generalData->fifoBufferSize + FILE_FRAME_HEADER_SIZE, fnum-firstMeasurementIndex);
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,9 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
|
||||
dataspace(0),
|
||||
dataset(0),
|
||||
datatype(PredType::STD_U16LE),
|
||||
nPixelsX(nx),
|
||||
nPixelsY(ny),
|
||||
numFramesInFile(0),
|
||||
|
||||
dataspace_para(0),
|
||||
|
||||
@ -35,11 +38,7 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
|
||||
|
||||
para2("bunch_id"),
|
||||
dataset_para2(0),
|
||||
datatype_para2(PredType::STD_U64LE),
|
||||
|
||||
nPixelsX(nx),
|
||||
nPixelsY(ny),
|
||||
numFramesInFile(0)
|
||||
datatype_para2(PredType::STD_U64LE)
|
||||
{
|
||||
#ifdef VERBOSE
|
||||
PrintMembers();
|
||||
@ -65,29 +64,27 @@ void HDF5File::PrintMembers() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) {
|
||||
nPixelsX = nx;
|
||||
nPixelsY = ny;
|
||||
}
|
||||
|
||||
|
||||
slsReceiverDefs::fileFormat HDF5File::GetFileType() {
|
||||
return HDF5;
|
||||
}
|
||||
|
||||
|
||||
void HDF5File::UpdateDataType() {
|
||||
switch(*dynamicRange){
|
||||
case 16:
|
||||
datatype = PredType::STD_U16LE;
|
||||
break;
|
||||
case 32:
|
||||
datatype = PredType::STD_U32LE;
|
||||
break;
|
||||
default:
|
||||
datatype = PredType::STD_U8LE;
|
||||
break;
|
||||
case 16: datatype = PredType::STD_U16LE; break;
|
||||
case 32: datatype = PredType::STD_U32LE; break;
|
||||
default: datatype = PredType::STD_U8LE; break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int HDF5File::CreateFile(uint64_t fnum) {
|
||||
numFramesInFile = 0;
|
||||
currentFileName = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
|
||||
@ -110,23 +107,17 @@ int HDF5File::CreateFile(uint64_t fnum) {
|
||||
}
|
||||
pthread_mutex_unlock(&Mutex);
|
||||
printf("%d HDF5 File: %s\n", index, currentFileName.c_str());
|
||||
|
||||
//virtual file
|
||||
if (master && (*detIndex==0))
|
||||
return CreateVirtualFile(fnum);
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
void HDF5File::CloseCurrentFile() {
|
||||
pthread_mutex_lock(&Mutex);
|
||||
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
|
||||
if (master && (*detIndex==0)) {
|
||||
HDF5FileStatic::CloseVirtualDataFile(virtualfd);
|
||||
}
|
||||
pthread_mutex_unlock(&Mutex);
|
||||
}
|
||||
|
||||
|
||||
void HDF5File::CloseAllFiles() {
|
||||
pthread_mutex_lock(&Mutex);
|
||||
HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
|
||||
@ -139,7 +130,7 @@ void HDF5File::CloseAllFiles() {
|
||||
|
||||
|
||||
int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
|
||||
if (numFramesInFile >= maxFramesPerFile) {/**max multiply by 100?????????????*/
|
||||
if (numFramesInFile >= maxFramesPerFile) {
|
||||
CloseCurrentFile();
|
||||
CreateFile(fnum);
|
||||
}
|
||||
@ -168,6 +159,7 @@ int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
|
||||
int HDF5File::CreateMasterFile(bool en, uint32_t size,
|
||||
uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap) {
|
||||
if (master && (*detIndex==0)) {
|
||||
virtualfd = 0;
|
||||
masterFileName = HDF5FileStatic::CreateMasterFileName(filePath, fileNamePrefix, *fileIndex);
|
||||
printf("Master File: %s\n", masterFileName.c_str());
|
||||
pthread_mutex_lock(&Mutex);
|
||||
@ -180,77 +172,29 @@ int HDF5File::CreateMasterFile(bool en, uint32_t size,
|
||||
}
|
||||
|
||||
|
||||
int HDF5File::CreateVirtualFile(uint64_t fnum) {
|
||||
void HDF5File::EndofAcquisition(uint64_t numf) {
|
||||
//not created before
|
||||
if (!virtualfd)
|
||||
CreateVirtualFile(numf);
|
||||
}
|
||||
|
||||
|
||||
int HDF5File::CreateVirtualFile(uint64_t numf) {
|
||||
if (master && (*detIndex==0)) {
|
||||
|
||||
//file name
|
||||
string virtualFileName = HDF5FileStatic::CreateVirtualFileName(filePath, fileNamePrefix, *fileIndex, *frameIndexEnable, fnum);
|
||||
printf("Virtual File: %s\n", virtualFileName.c_str());
|
||||
|
||||
//source file names
|
||||
int numReadouts = numDetX * numDetY;
|
||||
string fileNames[numReadouts];
|
||||
for (int i = 0; i < numReadouts; ++i) {
|
||||
fileNames[i] = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
|
||||
*frameIndexEnable, fnum, *detIndex, *numUnitsPerDetector, i);
|
||||
#ifdef VERBOSE
|
||||
printf("%d: File Name: %s\n", i, fileNames[i].c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
//datatype
|
||||
hid_t cdatatype;
|
||||
switch(*dynamicRange){
|
||||
case 16:
|
||||
cdatatype = H5T_STD_U16LE;
|
||||
break;
|
||||
case 32:
|
||||
cdatatype = H5T_STD_U32LE;
|
||||
break;
|
||||
default:
|
||||
cdatatype = H5T_STD_U8LE;
|
||||
break;
|
||||
}
|
||||
|
||||
//source dataset name
|
||||
ostringstream osfn;
|
||||
osfn << "/data";
|
||||
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
|
||||
string srcDatasetName = osfn.str();
|
||||
|
||||
//virtual dataset name
|
||||
osfn.str(""); osfn.clear();
|
||||
osfn << "/virtual_data";
|
||||
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
|
||||
string virtualDatasetName = osfn.str();
|
||||
//parameter 1 name
|
||||
osfn.str(""); osfn.clear();
|
||||
osfn << "/virtual_" << para1;
|
||||
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
|
||||
string vpara1DatasetName = osfn.str();
|
||||
|
||||
//parameter 2 name
|
||||
osfn.str(""); osfn.clear();
|
||||
osfn << "/virtual_" << para2;
|
||||
if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
|
||||
string vpara2DatasetName = osfn.str();
|
||||
|
||||
uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum);
|
||||
|
||||
//create virtual file
|
||||
pthread_mutex_lock(&Mutex);
|
||||
int ret = HDF5FileStatic::CreateVirtualDataFile(virtualfd, virtualFileName, virtualDatasetName, srcDatasetName,
|
||||
para1, vpara1DatasetName, H5T_STD_U32LE,
|
||||
para2, vpara2DatasetName, H5T_STD_U64LE,
|
||||
numReadouts, fileNames, *overWriteEnable, cdatatype,
|
||||
framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
|
||||
framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), HDF5_WRITER_VERSION);
|
||||
|
||||
if (ret == OK)
|
||||
ret = HDF5FileStatic::LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, vpara1DatasetName, vpara2DatasetName);
|
||||
|
||||
int ret = HDF5FileStatic::CreateVirtualDataFile(
|
||||
virtualfd, masterFileName,
|
||||
filePath, fileNamePrefix, *fileIndex, *frameIndexEnable,
|
||||
*detIndex, *numUnitsPerDetector,
|
||||
maxFramesPerFile, numf,
|
||||
"data", para1, para2,
|
||||
datatype, datatype_para1, datatype_para2,
|
||||
numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
|
||||
HDF5_WRITER_VERSION);
|
||||
pthread_mutex_unlock(&Mutex);
|
||||
return ret;
|
||||
}
|
||||
return OK;
|
||||
}
|
||||
|
||||
|
||||
@ -536,6 +536,17 @@ void UDPStandardImplementation::stopReceiver(){
|
||||
while(DataProcessor::GetRunningMask()){
|
||||
usleep(5000);
|
||||
}
|
||||
|
||||
//create virtual file
|
||||
if (fileWriteEnable && fileFormatType == HDF5) {
|
||||
uint64_t maxFramescaught = 0;
|
||||
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
|
||||
maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
|
||||
}
|
||||
if (maxFramescaught)
|
||||
dataProcessor[0]->EndofAcquisition(maxFramescaught);
|
||||
}
|
||||
|
||||
while(DataStreamer::GetRunningMask()){
|
||||
usleep(5000);
|
||||
}
|
||||
@ -637,8 +648,13 @@ void UDPStandardImplementation::shutDownUDPSockets() {
|
||||
|
||||
|
||||
void UDPStandardImplementation::closeFiles() {
|
||||
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it)
|
||||
uint64_t maxFramescaught = 0;
|
||||
for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
|
||||
(*it)->CloseFiles();
|
||||
maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
|
||||
}
|
||||
if (maxFramescaught)
|
||||
dataProcessor[0]->EndofAcquisition(maxFramescaught);
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user