slsReceiver: HDF5: unlimited max dimension in x (#numImages), x dim gets incremented by #frames when it gets an extra image, removed virtual_ naming in parameter names, small cased parameter names

This commit is contained in:
maliakal_d 2018-07-18 14:05:31 +02:00
parent 40c96b5562
commit 3b6ead7783
5 changed files with 171 additions and 60 deletions

View File

@ -185,5 +185,8 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
/** Dataset array for parameters */
vector <DataSet*> dataset_para;
/** Number of Images (including extended during acquisition) */
uint64_t extNumImages;
};
#endif

View File

@ -189,8 +189,8 @@ public:
/**
* Write Parameter Arrays as datasets (to virtual file)
* @param ind self index
* @param dspace_para dataspace of parametrs
* @param fnum frame number
* @param dspace_para parameter dataspace
* @param fnum current frame number
* @param dset_para vector or dataset pointers of parameters
* @param rheader sls_receiver_header pointer
* @param parameterDataTypes parameter datatypes
@ -230,6 +230,47 @@ public:
}
/**
* Extend datasets in #images dimension (x dimension)
* @param ind self index
* @param dpace data space pointer address
* @param dset data set pointer
* @param dspace_para parameter dataspace address pointer
* @param dset dataset parameter pointer
* @param initialNumImages initial number of images
* @returns 0 for success and 1 for fail
*/
static int ExtendDataset(int ind, DataSpace*& dspace, DataSet* dset,
DataSpace*& dspace_para, vector <DataSet*> dset_para,
uint64_t initialNumImages) {
try{
Exception::dontPrint(); //to handle errors
hsize_t dims[3];
dspace->getSimpleExtentDims(dims);
dims[0] += initialNumImages;
dset->extend(dims);
delete dspace;
dspace = new DataSpace(dset->getSpace());
hsize_t dims_para[1] = {dims[0]};
for (unsigned int i = 0; i < dset_para.size(); ++i)
dset_para[i]->extend(dims_para);
delete dspace_para;
dspace_para = new DataSpace(dset_para[0]->getSpace());
}
catch(Exception error){
cprintf(RED,"Error in extending dataset in object %d\n",ind);
error.printError();
return 1;
}
return 0;
}
/**
* Create master file
* @param fname master file name
@ -418,7 +459,8 @@ public:
//dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
dspace = new DataSpace (3,srcdims);
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
dspace = new DataSpace (3,srcdims,srcdimsmax);
//dataset name
@ -432,18 +474,24 @@ public:
DSetCreatPropList plist;
int fill_value = -1;
plist.setFillValue(dtype, &fill_value);
//chunked dataset if greater than max_chunked_images
if(nDimx > maxchunkedimages){
hsize_t chunk_dims[3] ={maxchunkedimages, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
}
// always create chunked dataset as unlimited is only supported with chunked layout
hsize_t chunk_dims[3] ={maxchunkedimages, nDimy, nDimz};
plist.setChunk(3, chunk_dims);
dset = new DataSet (fd->createDataSet(dsetname.c_str(), dtype, *dspace, plist));
//create parameter datasets
hsize_t dims[1] = {nDimx};
dspace_para = new DataSpace (1,dims);
hsize_t dimsmax[1] = {H5S_UNLIMITED};
dspace_para = new DataSpace (1,dims,dimsmax);
// always create chunked dataset as unlimited is only supported with chunked layout
DSetCreatPropList paralist;
hsize_t chunkpara_dims[3] ={maxchunkedimages};
paralist.setChunk(1, chunkpara_dims);
for (unsigned int i = 0; i < parameterNames.size(); ++i){
DataSet* ds = new DataSet(fd->createDataSet(parameterNames[i], parameterDataTypes[i], *dspace_para));
DataSet* ds = new DataSet(fd->createDataSet(parameterNames[i],
parameterDataTypes[i], *dspace_para, paralist));
dset_para.push_back(ds);
}
}
@ -499,52 +547,77 @@ public:
//file
hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
if (dfal < 0)
return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating file access property for virtual file ")
+ virtualFileName + string("\n"));
if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in setting strong file close degree for virtual file ")
+ virtualFileName + string("\n"));
fd = H5Fcreate( virtualFileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
if (fd < 0)
return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating virtual file ") + virtualFileName + string("\n"));
//attributes - version
hid_t dataspace_attr = H5Screate (H5S_SCALAR);
if (dataspace_attr < 0)
return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating dataspace for attribute in virtual file ")
+ virtualFileName + string("\n"));
hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
if (attrid < 0)
return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating attribute in virtual file ")
+ virtualFileName + string("\n"));
double attr_data = version;
if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in writing attribute in virtual file ")
+ virtualFileName + string("\n"));
if (H5Aclose (attrid) < 0)
return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in closing attribute in virtual file ")
+ virtualFileName + string("\n"));
//virtual dataspace
hsize_t vdsdims[3] = {numf, numDety * nDimy, numDetz * nDimz};
hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
if (vdsDataspace < 0)
return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating virtual dataspace in virtual file ")
+ virtualFileName + string("\n"));
hsize_t vdsdims_para[2] = {numf, (unsigned int) numDety * numDetz};
hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
if (vdsDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating virtual dataspace (parameters) in virtual file ")
+ virtualFileName + string("\n"));
//fill values
hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl < 0)
return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating file creation properties in virtual file ")
+ virtualFileName + string("\n"));
int fill_value = -1;
if (H5Pset_fill_value (dcpl, GetDataTypeinC(dataType), &fill_value) < 0)
return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating fill value in virtual file ")
+ virtualFileName + string("\n"));
hid_t dcpl_para[parameterNames.size()];
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE);
if (dcpl_para[i] < 0)
return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating file creation properties (parameters) in virtual file ")
+ virtualFileName + string("\n"));
if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0)
return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating fill value (parameters) in virtual file ")
+ virtualFileName + string("\n"));
}
//hyperslab
@ -554,7 +627,8 @@ public:
uint64_t framesSaved = 0;
for (int j = 0; j < numMajorHyperslab; j++) {
uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numf-framesSaved);
uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile)
? maxFramesPerFile : (numf-framesSaved);
hsize_t offset[3] = {framesSaved, 0, 0};
hsize_t count[3] = {nDimx, nDimy, nDimz};
hsize_t offset_para[2] = {framesSaved, 0};
@ -568,7 +642,8 @@ public:
error = true;
break;
}
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET,
offset_para, NULL, count_para, NULL) < 0) {
cprintf(RED,"could not select hyperslab for parameters\n");
error = true;
break;
@ -594,23 +669,31 @@ public:
//source dataspace
hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax);
if (srcDataspace < 0)
return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating source dataspace in virtual file ")
+ virtualFileName + string("\n"));
hsize_t srcdims_para[1] = {nDimx};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED};
hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para);
if (srcDataspace_para < 0)
return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating source dataspace (parameters) in virtual file ")
+ virtualFileName + string("\n"));
//mapping
if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(), srcDatasetName.c_str(), srcDataspace) < 0) {
if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(),
srcDatasetName.c_str(), srcDataspace) < 0) {
cprintf(RED,"could not set mapping for paramter 1\n");
error = true;
break;
}
for (unsigned int k = 0; k < parameterNames.size(); ++k) {
if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(), parameterNames[k], srcDataspace_para) < 0) {
if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(),
parameterNames[k], srcDataspace_para) < 0) {
cprintf(RED,"could not set mapping for paramter %d\n", k);
error = true;
break;
@ -629,22 +712,30 @@ public:
framesSaved += nDimx;
}
if (error)
return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in mapping files in virtual file ")
+ virtualFileName + string("\n"));
//dataset
string virtualDatasetName = string("/virtual_") + srcDataseName;
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(), GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
string virtualDatasetName = srcDataseName;
hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(),
GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (vdsdataset < 0)
return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating virutal dataset in virtual file ")
+ virtualFileName + string("\n"));
//virtual parameter dataset
for (unsigned int i = 0; i < parameterNames.size(); ++i) {
hid_t vdsdataset_para = H5Dcreate2 (fd,
(string("/virtual_") + string (parameterNames[i])).c_str(),
GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para, H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT);
parameterNames[i],
GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para,
H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT);
if (vdsdataset_para < 0)
return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualFileName + string("\n"));
return CloseFileOnError(fd,
string("Error in creating virutal dataset (parameters) in virtual file ")
+ virtualFileName + string("\n"));
}
//close
@ -814,14 +905,14 @@ public:
//**paramter datasets**
for (unsigned int i = 0; i < parameterNames.size(); ++i){
hid_t vdset_para = H5Dopen2( vfd, (string("/virtual_") + string (parameterNames[i])).c_str(), H5P_DEFAULT);
hid_t vdset_para = H5Dopen2( vfd, (string (parameterNames[i])).c_str(), H5P_DEFAULT);
if (vdset_para < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in opening virtual parameter dataset to create link\n"));
}
sprintf(linkname, "/entry/data/%s",(string("/virtual_") + string (parameterNames[i])).c_str());
sprintf(linkname, "/entry/data/%s",(string (parameterNames[i])).c_str());
if(H5Lcreate_external( relative_virtualfname.c_str(), (string("/virtual_") + string (parameterNames[i])).c_str(),
if(H5Lcreate_external( relative_virtualfname.c_str(), (string (parameterNames[i])).c_str(),
mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
H5Fclose(mfd); mfd = 0;
return CloseFileOnError( vfd, string("Error in creating link to virtual parameter dataset\n"));

View File

@ -36,7 +36,9 @@ HDF5File::HDF5File(int ind, uint32_t* maxf,
nPixelsY(ny),
numFramesInFile(0),
numActualPacketsInFile(0),
numFilesinAcquisition(0)
numFilesinAcquisition(0),
dataspace_para(0),
extNumImages(0)
{
#ifdef VERBOSE
PrintMembers();
@ -45,46 +47,46 @@ HDF5File::HDF5File(int ind, uint32_t* maxf,
parameterNames.clear();
parameterDataTypes.clear();
parameterNames.push_back("Frame Number");
parameterNames.push_back("frame number");
parameterDataTypes.push_back(PredType::STD_U64LE);
parameterNames.push_back("Exp Length/ Sub Exposure Time");
parameterNames.push_back("exp length or sub exposure time");
parameterDataTypes.push_back(PredType::STD_U32LE);
parameterNames.push_back("Packets Caught");
parameterNames.push_back("packets caught");
parameterDataTypes.push_back(PredType::STD_U32LE);
parameterNames.push_back("Bunch Id");
parameterNames.push_back("bunch id");
parameterDataTypes.push_back(PredType::STD_U64LE);
parameterNames.push_back("Time Stamp");
parameterNames.push_back("timestamp");
parameterDataTypes.push_back(PredType::STD_U64LE);
parameterNames.push_back("Mod Id");
parameterNames.push_back("mod id");
parameterDataTypes.push_back(PredType::STD_U16LE);
parameterNames.push_back("X Coord");
parameterNames.push_back("x Coord");
parameterDataTypes.push_back(PredType::STD_U16LE);
parameterNames.push_back("Y Coord");
parameterNames.push_back("y Coord");
parameterDataTypes.push_back(PredType::STD_U16LE);
parameterNames.push_back("Z Coord");
parameterNames.push_back("z Coord");
parameterDataTypes.push_back(PredType::STD_U16LE);
parameterNames.push_back("Debug");
parameterNames.push_back("debug");
parameterDataTypes.push_back(PredType::STD_U32LE);
parameterNames.push_back("Round Robin Number");
parameterNames.push_back("round robin number");
parameterDataTypes.push_back(PredType::STD_U16LE);
parameterNames.push_back("Detector Type");
parameterNames.push_back("detector type");
parameterDataTypes.push_back(PredType::STD_U8LE);
parameterNames.push_back("Detector Header Version");
parameterNames.push_back("detector header version");
parameterDataTypes.push_back(PredType::STD_U8LE);
parameterNames.push_back("Packets Caught Bit Mask");
parameterNames.push_back("packets caught bit mask");
StrType strdatatype(PredType::C_S1, MAX_NUM_PACKETS);
parameterDataTypes.push_back(strdatatype);
@ -141,8 +143,8 @@ int HDF5File::CreateFile(uint64_t fnum) {
if(!fnum) UpdateDataType();
uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
(((*numImages - fnum) > (*maxFramesPerFile)) ? // save up to maximum at a time
(*maxFramesPerFile) : (*numImages-fnum)));
(((extNumImages - fnum) > (*maxFramesPerFile)) ? // save up to maximum at a time
(*maxFramesPerFile) : (extNumImages-fnum)));
pthread_mutex_lock(&Mutex);
if (HDF5FileStatic::CreateDataFile(index, *overWriteEnable, currentFileName, (*numImages > 1),
fnum, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
@ -192,6 +194,20 @@ int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t
numFramesInFile++;
numActualPacketsInFile += nump;
pthread_mutex_lock(&Mutex);
// extend dataset (when receiver start followed by many status starts (jungfrau)))
if (fnum >= extNumImages) {
if (HDF5FileStatic::ExtendDataset(index, dataspace, dataset,
dataspace_para, dataset_para, *numImages) == OK) {
if (!silentMode) {
cprintf(BLUE,"%d Extending HDF5 dataset by %llu, Total x Dimension: %llu\n",
index, (long long unsigned int)extNumImages,
(long long unsigned int)(extNumImages + *numImages));
}
extNumImages += *numImages;
}
}
if (HDF5FileStatic::WriteDataFile(index, buffer + sizeof(sls_receiver_header),
// infinite then no need for %maxframesperfile
((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)),
@ -220,6 +236,7 @@ int HDF5File::CreateMasterFile(bool en, uint32_t size,
//beginning of every acquisition
numFramesInFile = 0;
numActualPacketsInFile = 0;
extNumImages = *numImages;
if (master && (*detIndex==0)) {
virtualfd = 0;

View File

@ -446,7 +446,7 @@ void UDPBaseImplementation::setFilePath(const char c[]){
else
FILE_LOG(logERROR) << "FilePath does not exist: " << filePath;
}
FILE_LOG(logINFO) << "Info: File path: " << filePath;
FILE_LOG(logINFO) << "File path: " << filePath;
}
void UDPBaseImplementation::setFileIndex(const uint64_t i){

View File

@ -542,7 +542,7 @@ void UDPStandardImplementation::stopReceiver(){
tot += dataProcessor[i]->GetNumFramesCaught();
uint64_t missingpackets = numberOfFrames*generalData->packetsPerFrame-listener[i]->GetPacketsCaught();
if (missingpackets) {
if ((int)missingpackets > 0) {
cprintf(RED, "\n[Port %d]\n",udpPortNum[i]);
cprintf(RED, "Missing Packets\t\t: %lld\n",(long long int)missingpackets);
cprintf(RED, "Complete Frames\t\t: %lld\n",(long long int)dataProcessor[i]->GetNumFramesCaught());