diff --git a/slsReceiverSoftware/include/DataStreamer.h b/slsReceiverSoftware/include/DataStreamer.h index 26dc19249..0bac8ee8c 100644 --- a/slsReceiverSoftware/include/DataStreamer.h +++ b/slsReceiverSoftware/include/DataStreamer.h @@ -82,7 +82,7 @@ class DataStreamer : private virtual slsReceiverDefs, public ThreadObject { /** * Reset parameters for new measurement (eg. for each scan) */ - void ResetParametersforNewMeasurement(); + void ResetParametersforNewMeasurement(char* fname); /** * Set GeneralData pointer to the one given @@ -237,5 +237,8 @@ class DataStreamer : private virtual slsReceiverDefs, public ThreadObject { /** Frame Number of First Frame for each real time acquisition (eg. for each scan) */ uint64_t firstMeasurementIndex; + + /* File name to stream */ + char fileNametoStream[MAX_STR_LENGTH]; }; diff --git a/slsReceiverSoftware/include/GeneralData.h b/slsReceiverSoftware/include/GeneralData.h index cab47a435..d58671040 100644 --- a/slsReceiverSoftware/include/GeneralData.h +++ b/slsReceiverSoftware/include/GeneralData.h @@ -94,13 +94,14 @@ public: * Get Header Infomation (frame number, packet number) * @param index thread index for debugging purposes * @param packetData pointer to data + * @param dynamicRange dynamic range to assign subframenumber if 32 bit mode * @param frameNumber frame number * @param packetNumber packet number * @param subFrameNumber sub frame number if applicable * @param bunchId bunch id */ - virtual void GetHeaderInfo(int index, char* packetData, uint64_t& frameNumber, - uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t bunchId) const + virtual void GetHeaderInfo(int index, char* packetData, uint32_t dynamicRange, + uint64_t& frameNumber, uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t& bunchId) const { subFrameNumber = -1; bunchId = -1; @@ -388,13 +389,14 @@ private: * Get Header Infomation (frame number, packet number) * @param index thread index for debugging purposes * @param packetData pointer to data + * @param dynamicRange dynamic range to assign subframenumber if 32 bit mode * @param frameNumber frame number * @param packetNumber packet number * @param subFrameNumber sub frame number if applicable * @param bunchId bunch id */ - void GetHeaderInfo(int index, char* packetData, uint64_t& frameNumber, - uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t bunchId) const + void GetHeaderInfo(int index, char* packetData, uint32_t dynamicRange, + uint64_t& frameNumber, uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t& bunchId) const { subFrameNumber = -1; jfrau_packet_header_t* header = (jfrau_packet_header_t*)(packetData); @@ -449,7 +451,7 @@ private: packetsPerFrame = 256; imageSize = dataSize*packetsPerFrame; frameIndexMask = 0xffffff; - maxFramesPerFile = 5;//EIGER_MAX_FRAMES_PER_FILE; + maxFramesPerFile = EIGER_MAX_FRAMES_PER_FILE; fifoBufferSize = imageSize; fifoBufferHeaderSize= FIFO_HEADER_NUMBYTES + FILE_FRAME_HEADER_SIZE; defaultFifoDepth = 100; @@ -475,19 +477,23 @@ private: * Get Header Infomation (frame number, packet number) * @param index thread index for debugging purposes * @param packetData pointer to data + * @param dynamicRange dynamic range to assign subframenumber if 32 bit mode * @param frameNumber frame number * @param packetNumber packet number * @param subFrameNumber sub frame number if applicable * @param bunchId bunch id */ - void GetHeaderInfo(int index, char* packetData, uint64_t& frameNumber, - uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t bunchId) const { + void GetHeaderInfo(int index, char* packetData, uint32_t dynamicRange, + uint64_t& frameNumber, uint32_t& packetNumber, uint32_t& subFrameNumber, uint64_t& bunchId) const { bunchId = -1; + subFrameNumber = -1; eiger_packet_footer_t* footer = (eiger_packet_footer_t*)(packetData + footerOffset); frameNumber = (uint64_t)((*( (uint64_t*) footer)) & frameIndexMask); packetNumber = (uint32_t)(*( (uint16_t*) footer->packetNumber))-1; - eiger_packet_header_t* header = (eiger_packet_header_t*) (packetData); - subFrameNumber = (uint64_t) *( (uint32_t*) header->subFrameNumber); + if (dynamicRange == 32) { + eiger_packet_header_t* header = (eiger_packet_header_t*) (packetData); + subFrameNumber = (uint64_t) *( (uint32_t*) header->subFrameNumber); + } } /** diff --git a/slsReceiverSoftware/include/HDF5File.h b/slsReceiverSoftware/include/HDF5File.h index 85dd99363..1eccff712 100644 --- a/slsReceiverSoftware/include/HDF5File.h +++ b/slsReceiverSoftware/include/HDF5File.h @@ -144,6 +144,27 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt /** Datatype of dataset */ DataType datatype; + /** Dataspace of parameters */ + DataSpace* dataspace_para; + + /** parameter1 */ + std::string para1; + + /** Dataset of parameter1 */ + DataSet* dataset_para1; + + /** Datatype of parameter1 */ + DataType datatype_para1; + + /** parameter2 */ + std::string para2; + + /** Dataset of parameter2 */ + DataSet* dataset_para2; + + /** Datatype of parameter2 */ + DataType datatype_para2; + /** Number of pixels in x direction */ uint32_t nPixelsX; diff --git a/slsReceiverSoftware/include/HDF5FileStatic.h b/slsReceiverSoftware/include/HDF5FileStatic.h index 47657bfb6..52f2ec511 100644 --- a/slsReceiverSoftware/include/HDF5FileStatic.h +++ b/slsReceiverSoftware/include/HDF5FileStatic.h @@ -104,13 +104,16 @@ public: * @param ds dataset pointer */ - static void CloseDataFile(int ind, H5File*& fd, DataSpace*& dp, DataSet*& ds) + static void CloseDataFile(int ind, H5File*& fd, DataSpace*& dp, DataSet*& ds, + DataSet*& ds_p1, DataSet*& ds_p2) { try { Exception::dontPrint(); //to handle errors - if(dp) {delete dp; dp = 0;} - if(ds) {delete ds; ds = 0;} - if(fd) {delete fd; fd = 0;} + if(dp) {delete dp; dp = 0;} + if(ds) {delete ds; ds = 0;} + if(ds_p1) {delete ds_p1; ds_p1 = 0;} + if(ds_p2) {delete ds_p2; ds_p2 = 0;} + if(fd) {delete fd; fd = 0;} } catch(Exception error) { cprintf(RED,"Error in closing HDF5 handles\n"); error.printError(); @@ -182,6 +185,34 @@ public: } + + /** + * Write Parameter Arrays as datasets (to virtual file) + */ + template + static int WriteParameterDatasets(int ind, DataSpace* dspace_para, uint64_t fnum, + DataSet*& dset_para1, DataType dtype_para1, P1* para1, + DataSet*& dset_para2, DataType dtype_para2, P2* para2) + { + hsize_t count[1] = {1}; + hsize_t start[1] = {fnum}; + hsize_t dims2[2] = {1}; + try{ + Exception::dontPrint(); //to handle errors + dspace_para->selectHyperslab( H5S_SELECT_SET, count, start); + DataSpace memspace(H5S_SCALAR); + dset_para1->write(para1, dtype_para1, memspace, *dspace_para); + dset_para2->write(para2, dtype_para2, memspace, *dspace_para); + } + catch(Exception error){ + cprintf(RED,"Error in writing parameters to file in object %d\n",ind); + error.printError(); + return 1; + } + return 0; + } + + /** * Create master file * @param fname master file name @@ -307,12 +338,22 @@ public: * @param dset dataset pointer * @param version version of software for hdf5 writing * @param maxchunkedimages maximum chunked images + * @param dspace_para dataspace of parameters + * @param para1 parameter 1 name + * @param dset_para1 dataset of parameter 1 + * @param dtype_para1 datatype of parameter 1 + * @param para2 parameter 2 name + * @param dset_para2 dataset of parameter 2 + * @param dtype_para2 datatype of parameter 2 * @returns 0 for success and 1 for fail */ static int CreateDataFile(int ind, bool owenable, string fname, bool frindexenable, uint64_t fnum, uint64_t nDimx, uint32_t nDimy, uint32_t nDimz, DataType dtype, H5File*& fd, DataSpace*& dspace, DataSet*& dset, - double version, uint64_t maxchunkedimages) + double version, uint64_t maxchunkedimages, + DataSpace*& dspace_para, + string para1, DataSet*& dset_para1, DataType dtype_para1, + string para2, DataSet*& dset_para2, DataType dtype_para2) { try { Exception::dontPrint(); //to handle errors @@ -351,6 +392,11 @@ public: }else dset = new DataSet (fd->createDataSet(dsetname.c_str(), dtype, *dspace)); + //create parameter datasets + hsize_t dims[1] = {nDimx}; + dspace_para = new DataSpace (1,dims); + dset_para1 = new DataSet(fd->createDataSet(para1.c_str(), dtype_para1, *dspace_para)); + dset_para2 = new DataSet(fd->createDataSet(para2.c_str(), dtype_para2, *dspace_para)); } catch(Exception error){ cprintf(RED,"Error in creating HDF5 handles in object %d\n",ind); @@ -366,13 +412,19 @@ public: /** * Create virtual file * (in C because H5Pset_virtual doesnt exist yet in C++) + * @param fd virtual file handle * @param virtualfname virtual file name * @param virtualDatasetname virtual dataset name * @param srcDatasetname source dataset name + * @param spara1Datasetname source dataset name of parameter 1 + * @param vpara1DatasetName virtual dataset name of parameter 1 + * @param dtype_para1 datatype of parameter 1 + * @param spara1Datasetname source dataset name of parameter 1 + * @param vpara2DatasetName virtual dataset name of parameter 2 + * @param dtype_para2 datatype of parameter 2 * @param numFiles number of files * @param fileNames array of file names * @param owenable overwrite enable - * @param fnum current frame number * @param dtype datatype * @param srcNDimx source number of objects in x dimension (Number of images) * @param srcNDimy source number of objects in y dimension (Number of pixels in y dir) @@ -384,115 +436,222 @@ public: * @returns 0 for success and 1 for fail */ static int CreateVirtualDataFile(hid_t& fd, string virtualfname, string virtualDatasetname, string srcDatasetname, - int numFiles, string fileNames[], bool owenable, uint64_t fnum, hid_t dtype, + string spara1DatasetName, string vpara1DatasetName, hid_t dtype_para1, + string spara2DatasetName, string vpara2DatasetName, hid_t dtype_para2, + int numFiles, string fileNames[], bool owenable, hid_t dtype, uint64_t srcNDimx, uint32_t srcNDimy, uint32_t srcNDimz, uint64_t dstNDimx, uint32_t dstNDimy, uint32_t dstNDimz, double version) { - //file hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); - if (dfal >= 0) { - if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) >= 0) { - if(!owenable) fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal); - else fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal); - if (fd >= 0) { + if (dfal < 0) + return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualfname + string("\n")); + if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) + return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualfname + string("\n")); + if(!owenable) + fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal); + else + fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal); + if (fd < 0) + return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualfname + string("\n")); - //attributes - version - hid_t dataspace_attr = H5Screate (H5S_SCALAR); - if (dataspace_attr >= 0) { - hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT); - if (attrid >= 0) { - double attr_data = version; - if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) >= 0) { - if (H5Aclose (attrid) >= 0) { - //dataspace - hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz}; - hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL); - if (vdsDataspace >= 0) { - hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz}; - hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL); - if (srcDataspace >= 0) { + //attributes - version + hid_t dataspace_attr = H5Screate (H5S_SCALAR); + if (dataspace_attr < 0) + return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualfname + string("\n")); + hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT); + if (attrid < 0) + return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualfname + string("\n")); + double attr_data = version; + if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0) + return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualfname + string("\n")); + if (H5Aclose (attrid) < 0) + return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualfname + string("\n")); - //fill values - hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE); - if (dcpl >= 0) { - int fill_value = -1; - if (H5Pset_fill_value (dcpl, dtype, &fill_value) >= 0) { - //hyperslab - hsize_t offset[3]={0,0,0},count[3]={srcNDimx,srcNDimy, srcNDimz}; - bool error = false; - for (int i = 0; i < numFiles; i++) { - //cout<<"("<= dstNDimz){ - offset[2] = 0; - offset[1] += srcNDimy; - } - } - if (!error) { - //dataset - hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); - if (vdsdataset >= 0){ + //**data dataset** + //dataspace + hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz}; + hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL); + if (vdsDataspace < 0) + return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualfname + string("\n")); + hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz}; + hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL); + if (srcDataspace < 0) + return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualfname + string("\n")); - H5Sclose(vdsDataspace);vdsDataspace = 0; - H5Sclose(srcDataspace);srcDataspace = 0; - H5Dclose(vdsdataset);vdsdataset = 0; - H5Fclose(fd);fd = 0; - return 0; + //fill values + hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE); + if (dcpl < 0) + return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualfname + string("\n")); + int fill_value = -1; + if (H5Pset_fill_value (dcpl, dtype, &fill_value) < 0) + return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualfname + string("\n")); - } else cprintf(RED, "could not create virtual dataset in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not map files in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not fill values in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create dcpl in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create source dataspace in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create virtual dataspace in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not close attribute in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not write attribute in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create attribute in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create dataspace for attribute in virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not set strong file close degree for virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not create dfal for virtual file %s\n", virtualfname.c_str()); - H5Fclose(fd); - fd = 0; - return 1; + //hyperslab + hsize_t offset[3]={0, 0, 0}, + count[3]={srcNDimx, srcNDimy, srcNDimz}; + bool error = false; + for (int i = 0; i < numFiles; i++) { + if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) { + cprintf(RED,"could not select hyperslab\n"); + error = true; + break; + } + if (H5Pset_virtual(dcpl, vdsDataspace, fileNames[i].c_str(), srcDatasetname.c_str(), srcDataspace) < 0) { + cprintf(RED,"could not set mapping\n"); + error = true; + break; + } + offset[2] += srcNDimz; + if (offset[2] >= dstNDimz) { + offset[2] = 0; + offset[1] += srcNDimy; + } + } + if (error) + return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualfname + string("\n")); + + //dataset + hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if (vdsdataset < 0) + return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualfname + string("\n")); + + //close + H5Sclose(vdsDataspace); + H5Sclose(srcDataspace); + H5Dclose(vdsdataset); + + + + //**paramter datasets** + //dataspace + hsize_t vdsdims_para[2] = {dstNDimx, numFiles}; + hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL); + if (vdsDataspace_para < 0) + return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualfname + string("\n")); + hsize_t srcdims_para[1] = {srcNDimx}; + hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL); + if (srcDataspace_para < 0) + return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualfname + string("\n")); + + //fill values + hid_t dcpl1 = H5Pcreate (H5P_DATASET_CREATE); + hid_t dcpl2 = H5Pcreate (H5P_DATASET_CREATE); + if ((dcpl1 < 0) || (dcpl2 < 0)) + return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualfname + string("\n")); + fill_value = -1; + if ((H5Pset_fill_value (dcpl1, dtype_para1, &fill_value) < 0) || (H5Pset_fill_value (dcpl2, dtype_para2, &fill_value) < 0)) + return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualfname + string("\n")); + + //hyperslab + hsize_t offset_para[2] = {0,0}, + count_para[2] = {dstNDimx,1}; + error = false; + for (int i = 0; i < numFiles; i++) { + //cout<<"("< - static int CopyVirtualFile(bool owenable, string oldFileName, string oldDatasetName, - string newFileName, string newDatasetName, - uint64_t nDimx, uint32_t nDimy, uint32_t nDimz, T datatype) + static int CopyVirtualFile(T datatype, bool owenable, string oldFileName, string oldDatasetName, + string newFileName, string newDatasetName, int rank, + uint64_t nDimx, uint32_t nDimy, uint32_t nDimz=0) { - - T *data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy*nDimz)); + T *data_out; + switch (rank) { + case 2: + data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy)); + break; + case 3: + data_out = (T*)malloc(sizeof(T)*(nDimx*nDimy*nDimz)); + break; + default: + cprintf(RED,"invalid rank. Options: 2 or 3\n"); + return 0; + } + if (datatype == PredType::STD_U16LE) { + printf("datatype:16\n"); + } else if (datatype == PredType::STD_U32LE) { + printf("datatype:32\n"); + } else if (datatype == PredType::STD_U64LE) { + printf("datatype:64\n"); + } else if (datatype == PredType::STD_U8LE) { + printf("datatype:8\n"); + } else { + cprintf(RED, "unknown datatype\n"); + return 1; + } + printf("owenable:%d\n" + "oldFileName:%s\n" + "oldDatasetName:%s\n" + "newFileName:%s\n" + "newDatasetName:%s\n" + "rank:%d\n" + "nDimx:%llu\n" + "nDimy:%u\n" + "nDimz:%u\n", + owenable?1:0, + oldFileName.c_str(), + oldDatasetName.c_str(), + newFileName.c_str(), + newDatasetName.c_str(), + rank, + (long long unsigned int)nDimx, + nDimy, + nDimz); H5File* oldfd; H5File* newfd; @@ -512,8 +671,14 @@ public: else newfd = new H5File( newFileName.c_str(), H5F_ACC_TRUNC, NULL, fapl ); //dataspace and dataset - hsize_t dims[3] = {nDimx, nDimy, nDimz}; - DataSpace* newDataspace = new DataSpace (3,dims); + DataSpace* newDataspace; + if (rank == 3) { + hsize_t dims[3] = {nDimx, nDimy, nDimz}; + newDataspace = new DataSpace (3,dims); + } else if (rank == 2) { + hsize_t dims[2] = {nDimx, nDimy}; + newDataspace = new DataSpace (2,dims); + } DataSet* newDataset = new DataSet( newfd->createDataSet(newDatasetName.c_str(), datatype, *newDataspace)); //write and close newDataset->write(data_out,datatype); @@ -537,46 +702,90 @@ public: /** * Link the Virtual File in the Master File * In C because H5Lcreate_external exists only in C + * @param masterFileName master file name * @param virtualfname virtual file name * @param virtualDatasetname virtual dataset name - * @param masterFileName master file name * @returns 0 for success and 1 for fail */ - int LinkVirtualInMaster(string virtualfname, string virtualDatasetname, string masterFileName) { + int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname, + string virtualpara1DatasetName, string virtualpara2DatasetName) { char linkname[100]; - sprintf(linkname, "/entry/data/%s",virtualDatasetname.c_str()); - hid_t mfd=-1, vfd=-1, vdset=-1; - hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); - if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) >= 0) { - mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal); - if (mfd >= 0) { - vfd = H5Fopen( virtualfname.c_str(), H5F_ACC_RDWR, dfal); - if (vfd >= 0) { - vdset = H5Dopen2( vfd, virtualDatasetname.c_str(), H5P_DEFAULT); - if (vdset >= 0) { - if(H5Lcreate_external( virtualfname.c_str(), virtualDatasetname.c_str(), - mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) >= 0) { - H5Dclose(vdset); - H5Fclose(mfd); - H5Fclose(vfd); - return 0; - cprintf(BLUE,"link created!\n"); - } else cprintf(RED, "could not create link\n"); - } else cprintf(RED, "could not open virtual dataset %s\n", virtualDatasetname.c_str()); - } else cprintf(RED, "could not open virtual file %s\n", virtualfname.c_str()); - } else cprintf(RED, "could not open master file %s\n", masterFileName.c_str()); - }else cprintf(RED, "could not set strong file close degree\n"); + hid_t vfd = 0; - if(vdset >=0 ) H5Dclose(vdset); - if(mfd >=0 ) H5Fclose(mfd); - if(vfd >=0 ) H5Fclose(vfd); - return 1; + hid_t dfal = H5Pcreate (H5P_FILE_ACCESS); + if (dfal < 0) + return CloseFileOnError( vfd, string("Error in creating file access property for link\n")); + if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0) + return CloseFileOnError( vfd, string("Error in setting strong file close degree for link\n")); + + //open master file + hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal); + if (mfd < 0) + return CloseFileOnError( vfd, string("Error in opening master file\n")); + + //open virtual file + vfd = H5Fopen( virtualfname.c_str(), H5F_ACC_RDWR, dfal); + if (vfd < 0) { + H5Fclose(mfd); mfd = 0; + return CloseFileOnError( vfd, string("Error in opening virtual file\n")); + } + + //**data dataset** + hid_t vdset = H5Dopen2( vfd, virtualDatasetname.c_str(), H5P_DEFAULT); + if (vdset < 0) { + H5Fclose(mfd); + return CloseFileOnError( vfd, string("Error in opening virtual data dataset\n")); + } + sprintf(linkname, "/entry/data/%s",virtualDatasetname.c_str()); + if(H5Lcreate_external( virtualfname.c_str(), virtualDatasetname.c_str(), + mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { + H5Fclose(mfd); mfd = 0; + return CloseFileOnError( vfd, string("Error in creating link to data dataset\n")); + } + H5Dclose(vdset); + + //**paramter datasets** + hid_t vdset1 = H5Dopen2( vfd, virtualpara1DatasetName.c_str(), H5P_DEFAULT); + hid_t vdset2 = H5Dopen2( vfd, virtualpara2DatasetName.c_str(), H5P_DEFAULT); + if ((vdset1 < 0) || (vdset2 < 0)) { + H5Fclose(mfd); mfd = 0; + return CloseFileOnError( vfd, string("Error in opening virtual parameter dataet to create link\n")); + } + sprintf(linkname, "/entry/data/%s",virtualpara1DatasetName.c_str()); + if(H5Lcreate_external( virtualfname.c_str(), virtualpara1DatasetName.c_str(), + mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { + H5Fclose(mfd); mfd = 0; + return CloseFileOnError( vfd, string("Error in creating link to virtual parameter 1 dataset\n")); + } + sprintf(linkname, "/entry/data/%s",virtualpara2DatasetName.c_str()); + if(H5Lcreate_external( virtualfname.c_str(), virtualpara2DatasetName.c_str(), + mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) { + H5Fclose(mfd); mfd = 0; + return CloseFileOnError( vfd, string("Error in creating link to virtual parameter 2 dataset\n")); + } + H5Dclose(vdset1); + H5Dclose(vdset2); + H5Fclose(mfd); mfd = 0; + H5Fclose(vfd); vfd = 0; + return 0; } + /** + * Print Error msg and Close File and called on error + * @returns 1 for fail + */ + static int CloseFileOnError(hid_t& fd, const string msg) { + cprintf(RED, "%s", msg.c_str()); + if(fd > 0) H5Fclose(fd); fd = 0; + return 1; + } }; + + + #endif diff --git a/slsReceiverSoftware/include/Listener.h b/slsReceiverSoftware/include/Listener.h index 60fcca248..fb3bca0ed 100644 --- a/slsReceiverSoftware/include/Listener.h +++ b/slsReceiverSoftware/include/Listener.h @@ -27,8 +27,9 @@ class Listener : private virtual slsReceiverDefs, public ThreadObject { * @param e ethernet interface * @param act pointer to activated * @param nf pointer to number of images to catch + * @param dr pointer to dynamic range */ - Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf); + Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf, uint32_t* dr); /** * Destructor @@ -236,6 +237,9 @@ class Listener : private virtual slsReceiverDefs, public ThreadObject { /** Number of Images to catch */ uint64_t* numImages; + /** Dynamic Range */ + uint32_t* dynamicRange; + /**Number of complete Packets caught for an entire acquisition (including all scans) */ uint64_t numTotalPacketsCaught; diff --git a/slsReceiverSoftware/src/DataStreamer.cpp b/slsReceiverSoftware/src/DataStreamer.cpp index 962643adf..21a2fc167 100644 --- a/slsReceiverSoftware/src/DataStreamer.cpp +++ b/slsReceiverSoftware/src/DataStreamer.cpp @@ -63,6 +63,7 @@ DataStreamer::DataStreamer(Fifo*& f, uint32_t* dr, uint32_t* freq, uint32_t* tim memset((void*)&timerBegin, 0, sizeof(timespec)); currentHeader = new char[255]; + strcpy(fileNametoStream, ""); } @@ -121,9 +122,11 @@ void DataStreamer::ResetParametersforNewAcquisition() { acquisitionStartedFlag = false; } -void DataStreamer::ResetParametersforNewMeasurement(){ +void DataStreamer::ResetParametersforNewMeasurement(char* fname){ firstMeasurementIndex = 0; measurementStartedFlag = false; + strcpy(fileNametoStream, fname); + cprintf(BLUE,"fname:%s\n",fname); CreateHeaderPart1(); } @@ -312,17 +315,15 @@ int DataStreamer::SendHeader(uint64_t fnum, uint32_t snum, bool dummy) { uint64_t frameIndex = -1; uint64_t acquisitionIndex = -1; uint32_t subframeIndex = -1; - char fname[MAX_STR_LENGTH] = "run"; char buf[1000] = ""; if (!dummy) { frameIndex = fnum - firstMeasurementIndex; acquisitionIndex = fnum - firstAcquisitionIndex; subframeIndex = snum; - /* fname to be included in fifo buffer? */ } - int len = sprintf(buf, jsonHeaderFormat, currentHeader, acquisitionIndex, frameIndex, subframeIndex, fname); + int len = sprintf(buf, jsonHeaderFormat, currentHeader, acquisitionIndex, frameIndex, subframeIndex, fileNametoStream); #ifdef VERBOSE printf("%d Streamer: buf:%s\n", index, buf); #endif diff --git a/slsReceiverSoftware/src/HDF5File.cpp b/slsReceiverSoftware/src/HDF5File.cpp index ed0867202..2c60bf176 100644 --- a/slsReceiverSoftware/src/HDF5File.cpp +++ b/slsReceiverSoftware/src/HDF5File.cpp @@ -26,6 +26,17 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex, dataspace(0), dataset(0), datatype(PredType::STD_U16LE), + + dataspace_para(0), + + para1("sub_frame_number"), + dataset_para1(0), + datatype_para1(PredType::STD_U32LE), + + para2("bunch_id"), + dataset_para2(0), + datatype_para2(PredType::STD_U64LE), + nPixelsX(nx), nPixelsY(ny), numFramesInFile(0) @@ -86,12 +97,14 @@ int HDF5File::CreateFile(uint64_t fnum) { if(!fnum) UpdateDataType(); uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum); - pthread_mutex_lock(&Mutex); if (HDF5FileStatic::CreateDataFile(index, *overWriteEnable, currentFileName, *frameIndexEnable, fnum, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), datatype, filefd, dataspace, dataset, - HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES) == FAIL) { + HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES, + dataspace_para, + para1, dataset_para1, datatype_para1, + para2, dataset_para2, datatype_para2) == FAIL) { pthread_mutex_unlock(&Mutex); return FAIL; } @@ -107,7 +120,7 @@ int HDF5File::CreateFile(uint64_t fnum) { void HDF5File::CloseCurrentFile() { pthread_mutex_lock(&Mutex); - HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset); + HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2); if (master && (*detIndex==0)) { HDF5FileStatic::CloseVirtualDataFile(virtualfd); } @@ -116,7 +129,7 @@ void HDF5File::CloseCurrentFile() { void HDF5File::CloseAllFiles() { pthread_mutex_lock(&Mutex); - HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset); + HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2); if (master && (*detIndex==0)) { HDF5FileStatic::CloseMasterDataFile(masterfd); HDF5FileStatic::CloseVirtualDataFile(virtualfd); @@ -126,17 +139,25 @@ void HDF5File::CloseAllFiles() { int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) { - if (numFramesInFile >= maxFramesPerFile) {/**max *100?????????????*/ + if (numFramesInFile >= maxFramesPerFile) {/**max multiply by 100?????????????*/ CloseCurrentFile(); CreateFile(fnum); } numFramesInFile++; + + uint32_t snum = (*((uint32_t*)(buffer + FILE_FRAME_HDR_FNUM_SIZE))); + uint64_t bid = (*((uint64_t*)(buffer + FILE_FRAME_HDR_FNUM_SIZE + FILE_FRAME_HDR_SNUM_SIZE))); pthread_mutex_lock(&Mutex); - if (HDF5FileStatic::WriteDataFile(index, buffer + FILE_FRAME_HEADER_SIZE, /** ignoring bunchid?????????? */ + if (HDF5FileStatic::WriteDataFile(index, buffer + FILE_FRAME_HEADER_SIZE, fnum%maxFramesPerFile, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), dataspace, dataset, datatype) == OK) { - pthread_mutex_unlock(&Mutex); - return OK; + if (HDF5FileStatic::WriteParameterDatasets(index, dataspace_para, + fnum%maxFramesPerFile, + dataset_para1, datatype_para1, &snum, + dataset_para2, datatype_para2, &bid) == OK) { + pthread_mutex_unlock(&Mutex); + return OK; + } } pthread_mutex_unlock(&Mutex); cprintf(RED,"%d Error: Write to file failed\n", index); @@ -202,18 +223,31 @@ int HDF5File::CreateVirtualFile(uint64_t fnum) { osfn << "/virtual_data"; if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum; string virtualDatasetName = osfn.str(); + //parameter 1 name + osfn.str(""); osfn.clear(); + osfn << "/virtual_" << para1; + if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum; + string vpara1DatasetName = osfn.str(); + + //parameter 2 name + osfn.str(""); osfn.clear(); + osfn << "/virtual_" << para2; + if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum; + string vpara2DatasetName = osfn.str(); uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum); //create virtual file pthread_mutex_lock(&Mutex); int ret = HDF5FileStatic::CreateVirtualDataFile(virtualfd, virtualFileName, virtualDatasetName, srcDatasetName, - numReadouts, fileNames, *overWriteEnable, fnum, cdatatype, + para1, vpara1DatasetName, H5T_STD_U32LE, + para2, vpara2DatasetName, H5T_STD_U64LE, + numReadouts, fileNames, *overWriteEnable, cdatatype, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), - framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),HDF5_WRITER_VERSION); + framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), HDF5_WRITER_VERSION); if (ret == OK) - ret = HDF5FileStatic::LinkVirtualInMaster(virtualFileName, virtualDatasetName, masterFileName); + ret = HDF5FileStatic::LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, vpara1DatasetName, vpara2DatasetName); pthread_mutex_unlock(&Mutex); return ret; diff --git a/slsReceiverSoftware/src/Listener.cpp b/slsReceiverSoftware/src/Listener.cpp index 3f07177f2..3eb04c5cc 100644 --- a/slsReceiverSoftware/src/Listener.cpp +++ b/slsReceiverSoftware/src/Listener.cpp @@ -29,7 +29,7 @@ pthread_mutex_t Listener::Mutex = PTHREAD_MUTEX_INITIALIZER; const GeneralData* Listener::generalData(0); -Listener::Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf) : +Listener::Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, uint64_t* nf, uint32_t* dr) : ThreadObject(NumberofListeners), fifo(f), acquisitionStartedFlag(false), @@ -40,6 +40,7 @@ Listener::Listener(Fifo*& f, runStatus* s, uint32_t* portno, char* e, int* act, eth(e), activated(act), numImages(nf), + dynamicRange(dr), numTotalPacketsCaught(0), numPacketsCaught(0), firstAcquisitionIndex(0), @@ -302,7 +303,7 @@ uint32_t Listener::ListenToAnImage(char* buf) { //look for carry over if (carryOverFlag) { //check if its the current image packet - generalData->GetHeaderInfo(index, carryOverPacket, fnum, pnum, snum, bid); + generalData->GetHeaderInfo(index, carryOverPacket, *dynamicRange, fnum, pnum, snum, bid); if (fnum != currentFrameIndex) { return generalData->imageSize; } @@ -327,7 +328,7 @@ uint32_t Listener::ListenToAnImage(char* buf) { //update parameters numPacketsCaught++; //record immediately to get more time before socket shutdown numTotalPacketsCaught++; - generalData->GetHeaderInfo(index, listeningPacket, fnum, pnum, snum, bid); + generalData->GetHeaderInfo(index, listeningPacket, *dynamicRange, fnum, pnum, snum, bid); lastCaughtFrameIndex = fnum; #ifdef VERBOSE if (!index && !pnum) cprintf(GREEN,"Listening %d: fnum:%lld, pnum:%d\n", index, (long long int)fnum, pnum); diff --git a/slsReceiverSoftware/src/UDPStandardImplementation.cpp b/slsReceiverSoftware/src/UDPStandardImplementation.cpp index 499d20460..cc6f4fba1 100644 --- a/slsReceiverSoftware/src/UDPStandardImplementation.cpp +++ b/slsReceiverSoftware/src/UDPStandardImplementation.cpp @@ -393,7 +393,7 @@ int UDPStandardImplementation::setDetectorType(const detectorType d) { //create threads for ( int i=0; i < numThreads; ++i ) { - listener.push_back(new Listener(fifo[i], &status, &udpPortNum[i], eth, &activated, &numberOfFrames)); + listener.push_back(new Listener(fifo[i], &status, &udpPortNum[i], eth, &activated, &numberOfFrames, &dynamicRange)); dataProcessor.push_back(new DataProcessor(fifo[i], &fileFormatType, &fileWriteEnable, &dataStreamEnable, &callbackAction, rawDataReadyCallBack,pRawDataReady)); if (Listener::GetErrorMask() || DataProcessor::GetErrorMask()) { @@ -780,8 +780,13 @@ void UDPStandardImplementation::ResetParametersforNewMeasurement() { (*it)->ResetParametersforNewMeasurement(); for (vector::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) (*it)->ResetParametersforNewMeasurement(); - for (vector::const_iterator it = dataStreamer.begin(); it != dataStreamer.end(); ++it) - (*it)->ResetParametersforNewMeasurement(); + + if (dataStreamEnable) { + char fnametostream[1000]; + sprintf(fnametostream, "%s/%s", filePath, fileName); + for (vector::const_iterator it = dataStreamer.begin(); it != dataStreamer.end(); ++it) + (*it)->ResetParametersforNewMeasurement(fnametostream); + } }