changed to have only one virtual file with bunch id, subfnum and data datasets all linked in master file

slsReceiverSoftware/include/DataProcessor.h

@@ -174,6 +174,11 @@ class DataProcessor : private virtual slsReceiverDefs, public ThreadObject {
 	 */
 	void CloseFiles();
 
+	/**
+	 * End of Acquisition
+	 * @param numf number of images caught
+	 */
+	void EndofAcquisition(uint64_t numf);
 
 
  private:

slsReceiverSoftware/include/File.h

@@ -141,7 +141,13 @@ class File : private virtual slsReceiverDefs {
 		cprintf(RED,"This is a generic function SetNumberofPixels that should be overloaded by a derived class\n");
 	}
 
-
+	/**
+	 * End of Acquisition
+	 * @param numf number of images caught
+	 */
+	virtual void EndofAcquisition(uint64_t numf) {
+		cprintf(RED,"This is a generic function EndofAcquisition that should be overloaded by a derived class\n");
+	}
 
  protected:
 

slsReceiverSoftware/include/HDF5File.h

@@ -101,10 +101,17 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
 			uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap);
 
 	/**
-	 * Create Virtual File
+	 * End of Acquisition
-	 * @param fnum frame number
+	 * @param numf number of images caught
 	 */
-	int CreateVirtualFile(uint64_t fnum);
+	void EndofAcquisition(uint64_t numf);
+
+	/**
+	 * Create Virtual File
+	 * @param numf number of images caught
+	 * @returns OK or FAIL
+	 */
+	int CreateVirtualFile(uint64_t numf);
 
 
  private:
@@ -122,7 +129,6 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
 
 
 
-
 	/** mutex to update static items among objects (threads)*/
 	static pthread_mutex_t Mutex;
 
@@ -144,6 +150,16 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
 	/** Datatype of dataset */
 	DataType datatype;
 
+	/** Number of pixels in x direction */
+	uint32_t nPixelsX;
+
+	/** Number of pixels in y direction */
+	uint32_t nPixelsY;
+
+	/** Number of frames in file */
+	uint32_t numFramesInFile;
+
+	//parameters
 	/** Dataspace of parameters */
 	DataSpace* dataspace_para;
 
@@ -165,14 +181,7 @@ class HDF5File : private virtual slsReceiverDefs, public File, public HDF5FileSt
 	/** Datatype of parameter2 */
 	DataType datatype_para2;
 
-	/** Number of pixels in x direction */
-	uint32_t nPixelsX;
 
-	/** Number of pixels in y direction */
-	uint32_t nPixelsY;
-
-	/** Number of frames in file */
-	uint32_t numFramesInFile;
 
 };
 #endif

slsReceiverSoftware/include/HDF5FileStatic.h

@@ -80,17 +80,13 @@ public:
 	 * @param fnameprefix file name prefix (includes scan and position variables)
 	 * @param fnum current frame number
 	 * @param findex file index
-	 * @param frindexenable frame index enable
-	 * @param fnum frame number index
 	 * @returns virtual file name
 	 */
-	static string CreateVirtualFileName(char* fpath, char* fnameprefix, uint64_t findex, bool frindexenable,
+	static string CreateVirtualFileName(char* fpath, char* fnameprefix, uint64_t findex)
-			uint64_t fnum)
 	{
 		ostringstream osfn;
 		osfn << fpath << "/" << fnameprefix;
 		osfn << "_virtual";
-		if (frindexenable) osfn << "_f" << setfill('0') << setw(12) << fnum;
 		osfn << "_" << findex;
 		osfn << ".h5";
 		return osfn.str();
@@ -408,185 +404,198 @@ public:
 	}
 
 
-
 	/**
 	 * Create virtual file
 	 * (in C because H5Pset_virtual doesnt exist yet in C++)
 	 * @param fd virtual file handle
-	 * @param virtualfname virtual file name
+	 * @param masterFileName master file name
-	 * @param virtualDatasetname virtual dataset name
+	 * @param fpath file path
-	 * @param srcDatasetname source dataset name
+	 * @param fnameprefix file name prefix (includes scan and position variables)
-	 * @param spara1Datasetname source dataset name of parameter 1
+	 * @param findex file index
-	 * @param vpara1DatasetName virtual dataset name of parameter 1
+	 * @param frindexenable frame index enable
-	 * @param dtype_para1 datatype of parameter 1
+	 * @param dindex readout index
-	 * @param spara1Datasetname source dataset name of parameter 1
+	 * @param numunits number of units per readout. eg. eiger has 2 udp units per readout
-	 * @param vpara2DatasetName virtual dataset name of parameter 2
+	 * @param maxFramesPerFile maximum frames per file
-	 * @param dtype_para2 datatype of parameter 2
+	 * @param numf number of frames caught
-	 * @param numFiles number of files
+	 * @param srcDataseName source dataset name
-	 * @param fileNames array of file names
+	 * @param srcP1DatasetName source parameter 1 dataset name
-	 * @param owenable overwrite enable
+	 * @param srcP2DatasetName source parameter 2 dataset name
-	 * @param dtype datatype
+	 * @param dataType datatype of data dataset
-	 * @param srcNDimx source number of objects in x dimension (Number of images)
+	 * @param p1DataType datatype of parameter 1
-	 * @param srcNDimy source number of objects in y dimension (Number of pixels in y dir)
+	 * @param p2DataType datatype of parameter 2
-	 * @param srcNDimz source number of objects in z dimension (Number of pixels in x dir)
+	 * @param numDety number of readouts in Y dir
-	 * @param dstNDimx destination number of objects in x dimension (Number of images)
+	 * @param numDetz number of readouts in Z dir
-	 * @param dstNDimy destination number of objects in y dimension (Number of pixels in y dir)
+	 * @param nDimy number of objects in y dimension in source file (Number of pixels in y dir)
-	 * @param dstNDimz destination number of objects in z dimension (Number of pixels in x dir)
+	 * @param nDimz number of objects in z dimension in source file (Number of pixels in x dir)
 	 * @param version version of software for hdf5 writing
 	 * @returns 0 for success and 1 for fail
 	 */
-	static int CreateVirtualDataFile(hid_t& fd, string virtualfname, string virtualDatasetname, string srcDatasetname,
+	static int CreateVirtualDataFile(
-			string spara1DatasetName, string vpara1DatasetName, hid_t dtype_para1,
+			hid_t& fd, string masterFileName,
-			string spara2DatasetName, string vpara2DatasetName, hid_t dtype_para2,
+			char* fpath, char* fnameprefix, uint64_t findex, bool frindexenable,
-			int numFiles, string fileNames[], bool owenable, hid_t dtype,
+			int dindex, int numunits,
-			uint64_t srcNDimx, uint32_t srcNDimy, uint32_t srcNDimz,
+			uint32_t maxFramesPerFile, uint64_t numf,
-			uint64_t dstNDimx, uint32_t dstNDimy, uint32_t dstNDimz, double version)
+			string srcDataseName, string srcP1DatasetName, string srcP2DatasetName,
+			DataType dataType, DataType p1DataType, DataType p2DataType,
+			int numDety, int numDetz, uint32_t nDimy, uint32_t nDimz,
+			double version)
 	{
+		//virtual names
+		string virtualFileName = CreateVirtualFileName(fpath, fnameprefix, findex);
+		printf("Virtual File: %s\n", virtualFileName.c_str());
+
 		//file
 		hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
 		if (dfal < 0)
-			return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating file access property for virtual file ") + virtualFileName + string("\n"));
 		if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
-			return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in setting strong file close degree for virtual file ") + virtualFileName + string("\n"));
-		if(!owenable)
+		fd = H5Fcreate( virtualFileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
-			fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, dfal);
-		else
-			fd = H5Fcreate( virtualfname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
 		if (fd < 0)
-			return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating virtual file ") + virtualFileName + string("\n"));
-
 
 		//attributes - version
 		hid_t dataspace_attr = H5Screate (H5S_SCALAR);
 		if (dataspace_attr < 0)
-			return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating dataspace for attribute in virtual file ") + virtualFileName + string("\n"));
 		hid_t attrid = H5Acreate2 (fd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
 		if (attrid < 0)
-			return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating attribute in virtual file ") + virtualFileName + string("\n"));
 		double attr_data = version;
 		if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
-			return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in writing attribute in virtual file ") + virtualFileName + string("\n"));
 				if (H5Aclose (attrid) < 0)
-			return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in closing attribute in virtual file ") + virtualFileName + string("\n"));
 
 
-
+		//virtual dataspace
-		//**data dataset**
+		hsize_t vdsdims[3] = {numf, numDety * nDimy, numDetz * nDimz};
-		//dataspace
-		hsize_t vdsdims[3] = {dstNDimx, dstNDimy, dstNDimz};
 		hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
 		if (vdsDataspace < 0)
-			return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating virtual dataspace in virtual file ") + virtualFileName + string("\n"));
-		hsize_t srcdims[3] = {srcNDimx, srcNDimy, srcNDimz};
+		hsize_t vdsdims_para[2] = {numf, numDety * numDetz};
-		hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
+		hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
-		if (srcDataspace < 0)
+		if (vdsDataspace_para < 0)
-			return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
+
 
 		//fill values
 		hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
 		if (dcpl < 0)
-			return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating file creation properties in virtual file ") + virtualFileName + string("\n"));
 		int fill_value = -1;
-		if (H5Pset_fill_value (dcpl, dtype, &fill_value) < 0)
+		if (H5Pset_fill_value (dcpl, GetDataTypeinC(dataType), &fill_value) < 0)
-			return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating fill value in virtual file ") + virtualFileName + string("\n"));
-
-
-		//hyperslab
-		hsize_t offset[3]={0, 0, 0},
-				count[3]={srcNDimx, srcNDimy, srcNDimz};
-		bool error = false;
-		for (int i = 0; i < numFiles; i++) {
-			if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
-				cprintf(RED,"could not select hyperslab\n");
-				error = true;
-				break;
-			}
-			if (H5Pset_virtual(dcpl, vdsDataspace, fileNames[i].c_str(), srcDatasetname.c_str(), srcDataspace) < 0) {
-				cprintf(RED,"could not set mapping\n");
-				error = true;
-				break;
-			}
-			offset[2] += srcNDimz;
-			if (offset[2] >=  dstNDimz) {
-				offset[2] = 0;
-				offset[1] += srcNDimy;
-			}
-		}
-		if (error)
-			return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualfname + string("\n"));
-
-		//dataset
-		hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetname.c_str(), dtype, vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
-		if (vdsdataset < 0)
-			return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualfname + string("\n"));
-
-		//close
-		H5Sclose(vdsDataspace);
-		H5Sclose(srcDataspace);
-		H5Dclose(vdsdataset);
-
-
-
-		//**paramter datasets**
-		//dataspace
-		hsize_t vdsdims_para[2] = {dstNDimx, numFiles};
-		hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
-		if (vdsDataspace_para < 0)
-			return CloseFileOnError(fd, string("Error in creating virtual dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
-		hsize_t srcdims_para[1] = {srcNDimx};
-		hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
-		if (srcDataspace_para < 0)
-			return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualfname + string("\n"));
-
-		//fill values
 		hid_t dcpl1 = H5Pcreate (H5P_DATASET_CREATE);
 		hid_t dcpl2 = H5Pcreate (H5P_DATASET_CREATE);
 		if ((dcpl1 < 0) || (dcpl2 < 0))
-			return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in creating file creation properties (parameters) in virtual file ") + virtualFileName + string("\n"));
 		fill_value = -1;
-		if ((H5Pset_fill_value (dcpl1, dtype_para1, &fill_value) < 0) || (H5Pset_fill_value (dcpl2, dtype_para2, &fill_value) < 0))
+		if ((H5Pset_fill_value (dcpl1, GetDataTypeinC(p1DataType), &fill_value) < 0) ||
-			return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualfname + string("\n"));
+				(H5Pset_fill_value (dcpl2, GetDataTypeinC(p2DataType), &fill_value) < 0))
+			return CloseFileOnError(fd, string("Error in creating fill value (parameters) in virtual file ") + virtualFileName + string("\n"));
+
 
 		//hyperslab
-		hsize_t offset_para[2] = {0,0},
+		int numMajorHyperslab = numf/maxFramesPerFile;
-				count_para[2] = {dstNDimx,1};
+		if (numf%maxFramesPerFile) numMajorHyperslab++;
-		error = false;
+		bool error = false;
-		for (int i = 0; i < numFiles; i++) {
+		uint64_t framesSaved = 0;
-			//cout<<"("<<offset[0]<<","<<offset[1]<<","<<offset[2]<<")"<<endl;
+		for (int j = 0; j < numMajorHyperslab; j++) {
-			if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
+
-				cprintf(RED,"could not select hyperslab\n");
+			uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile) ? maxFramesPerFile : (numf-framesSaved);
-				error = true;
+			hsize_t offset[3] 		= {framesSaved, 0, 0};
-				break;
+			hsize_t	count[3] 		= {nDimx, nDimy, nDimz};
+			hsize_t offset_para[2] 	= {framesSaved, 0};
+			hsize_t	count_para[2] 	= {nDimx, 1};
+
+			for (int i = 0; i < numDety * numDetz; ++i) {
+
+				//setect hyperslabs
+				if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
+					cprintf(RED,"could not select hyperslab\n");
+					error = true;
+					break;
+				}
+				if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET, offset_para, NULL, count_para, NULL) < 0) {
+					cprintf(RED,"could not select hyperslab for parameters\n");
+					error = true;
+					break;
+				}
+
+				//source file name
+				string srcFileName = HDF5FileStatic::CreateFileName(fpath, fnameprefix, findex,
+						frindexenable, framesSaved, dindex, numunits, i);
+
+				//source dataset name
+				ostringstream osfn;
+				osfn << "/data";
+				if (frindexenable) osfn << "_f" << setfill('0') << setw(12) << framesSaved;
+				string srcDatasetName = osfn.str();
+
+				//source dataspace
+				hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
+				hid_t srcDataspace = H5Screate_simple(3, srcdims, NULL);
+				if (srcDataspace < 0)
+					return CloseFileOnError(fd, string("Error in creating source dataspace in virtual file ") + virtualFileName + string("\n"));
+				hsize_t srcdims_para[1] = {nDimx};
+				hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, NULL);
+				if (srcDataspace_para < 0)
+					return CloseFileOnError(fd, string("Error in creating source dataspace (parameters) in virtual file ") + virtualFileName + string("\n"));
+
+				//mapping
+				if (H5Pset_virtual(dcpl, vdsDataspace, srcFileName.c_str(), srcDatasetName.c_str(), srcDataspace) < 0) {
+					cprintf(RED,"could not set mapping for paramter 1\n");
+					error = true;
+					break;
+				}
+
+				if (H5Pset_virtual(dcpl1, vdsDataspace_para, srcFileName.c_str(), srcP1DatasetName.c_str(), srcDataspace_para) < 0) {
+					cprintf(RED,"could not set mapping for paramter 1\n");
+					error = true;
+					break;
+				}
+				if (H5Pset_virtual(dcpl2, vdsDataspace_para, srcFileName.c_str(), srcP2DatasetName.c_str(), srcDataspace_para) < 0) {
+					cprintf(RED,"could not set mapping for paramter 2\n");
+					error = true;
+					break;
+				}
+				//H5Sclose(srcDataspace);
+				//H5Sclose(srcDataspace_para);
+				offset[2] += nDimz;
+				if (offset[2] >=  (numDetz * nDimz)) {
+					offset[2] = 0;
+					offset[1] += nDimy;
+				}
+				offset_para[1]++;
 			}
-			if (H5Pset_virtual(dcpl1, vdsDataspace_para, fileNames[i].c_str(), spara1DatasetName.c_str(), srcDataspace_para) < 0) {
+			framesSaved += nDimx;
-				cprintf(RED,"could not set mapping for paramter 1\n");
-				error = true;
-				break;
-			}
-			if (H5Pset_virtual(dcpl2, vdsDataspace_para, fileNames[i].c_str(), spara2DatasetName.c_str(), srcDataspace_para) < 0) {
-				cprintf(RED,"could not set mapping for paramter 2\n");
-				error = true;
-				break;
-			}
-			offset_para[1] ++;
 		}
 		if (error)
-			return CloseFileOnError(fd, string("Error in mapping files (parameters) in virtual file ") + virtualfname + string("\n"));
+			return CloseFileOnError(fd, string("Error in mapping files in virtual file ") + virtualFileName + string("\n"));
 
 		//dataset
-		hid_t vdsdataset1 = H5Dcreate2 (fd, vpara1DatasetName.c_str(), dtype_para1, vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT);
+		string virtualDatasetName = string("/virtual_") + srcDataseName;
-		hid_t vdsdataset2 = H5Dcreate2 (fd, vpara2DatasetName.c_str(), dtype_para2, vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT);
+		string virutalP1DatasetName = string("/virtual_") + srcP1DatasetName;
-		if ((vdsdataset1 < 0) && (vdsdataset1 < 0))
+		string virutalP2DatasetName = string("/virtual_") + srcP2DatasetName;
-			return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualfname + string("\n"));
+		hid_t vdsdataset = H5Dcreate2 (fd, virtualDatasetName.c_str(), GetDataTypeinC(dataType), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+		if (vdsdataset < 0)
+			return CloseFileOnError(fd, string("Error in creating virutal dataset in virtual file ") + virtualFileName + string("\n"));
+		hid_t vdsdataset1 = H5Dcreate2 (fd, virutalP1DatasetName.c_str(), GetDataTypeinC(p1DataType), vdsDataspace_para, H5P_DEFAULT, dcpl1, H5P_DEFAULT);
+		hid_t vdsdataset2 = H5Dcreate2 (fd, virutalP2DatasetName.c_str(), GetDataTypeinC(p2DataType), vdsDataspace_para, H5P_DEFAULT, dcpl2, H5P_DEFAULT);
+		if ((vdsdataset1 < 0) || (vdsdataset2 < 0))
+			return CloseFileOnError(fd, string("Error in creating virutal dataset (parameters) in virtual file ") + virtualFileName + string("\n"));
 
 		//close
+		H5Sclose(vdsDataspace);
 		H5Sclose(vdsDataspace_para);
-		H5Sclose(srcDataspace_para);
+		H5Dclose(vdsdataset);
 		H5Dclose(vdsdataset1);
 		H5Dclose(vdsdataset2);
-
 		H5Fclose(fd); fd = 0;
-		return 0;
+
+		//link
+		return LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, virutalP1DatasetName, virutalP2DatasetName);
 	}
 
 
@@ -699,15 +708,19 @@ public:
 		return 0;
 	}
 
+
+
 	/**
 	 * Link the Virtual File in the Master File
 	 * In C because H5Lcreate_external exists only in C
 	 * @param masterFileName master file name
 	 * @param virtualfname virtual file name
 	 * @param virtualDatasetname virtual dataset name
+	 * @param virtualpara1DatasetName virtual dataset name of parameter 1
+	 * @param virtualpara2DatasetName virtual dataset name of parameter 1
 	 * @returns 0 for success and 1 for fail
 	 */
-	int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname,
+	static int LinkVirtualInMaster(string masterFileName, string virtualfname, string virtualDatasetname,
 			string virtualpara1DatasetName, string virtualpara2DatasetName) {
 		char linkname[100];
 		hid_t vfd = 0;
@@ -746,10 +759,18 @@ public:
 
 		//**paramter datasets**
 		hid_t vdset1 = H5Dopen2( vfd, virtualpara1DatasetName.c_str(), H5P_DEFAULT);
+		if (vdset1 < 0) {
+			H5Fclose(mfd); mfd = 0;
+				return CloseFileOnError( vfd, string("Error in opening virtual parameter 1 dataet to create link\n"));
+		}
 		hid_t vdset2 = H5Dopen2( vfd, virtualpara2DatasetName.c_str(), H5P_DEFAULT);
+		if (vdset2 < 0) {
+			H5Fclose(mfd); mfd = 0;
+				return CloseFileOnError( vfd, string("Error in opening virtual parameter 2 dataet to create link\n"));
+		}
 		if ((vdset1 < 0) || (vdset2 < 0)) {
 			H5Fclose(mfd); mfd = 0;
-			return CloseFileOnError( vfd, string("Error in opening virtual parameter dataet to create link\n"));
+			return CloseFileOnError( vfd, string("Error in opening either virtual parameter dataet to create link\n"));
 		}
 		sprintf(linkname, "/entry/data/%s",virtualpara1DatasetName.c_str());
 		if(H5Lcreate_external( virtualfname.c_str(), virtualpara1DatasetName.c_str(),
@@ -783,6 +804,26 @@ public:
 	}
 
 
+	/**
+	 * Get Data type in C
+	 * @param dtype datatype in C++
+	 * @returns datatype in C
+	 */
+	static hid_t GetDataTypeinC(DataType dtype) {
+		if (dtype == PredType::STD_U8LE)
+			return H5T_STD_U8LE;
+		else if (dtype == PredType::STD_U16LE)
+			return H5T_STD_U16LE;
+		else if (dtype == PredType::STD_U32LE)
+			return H5T_STD_U32LE;
+		else if (dtype == PredType::STD_U64LE)
+			return H5T_STD_U64LE;
+		else {
+			cprintf(RED, "Invalid Data type\n");
+			return H5T_STD_U64LE;
+		}
+	}
+
 };
 
 

slsReceiverSoftware/src/DataProcessor.cpp

@@ -249,6 +249,12 @@ void DataProcessor::CloseFiles() {
 		file->CloseAllFiles();
 }
 
+void DataProcessor::EndofAcquisition(uint64_t numf) {
+	if (*fileWriteEnable && file->GetFileType() == HDF5 && numf) {
+		file->EndofAcquisition(numf);
+	}
+}
+
 
 void DataProcessor::ThreadExecution() {
 	char* buffer=0;
@@ -307,7 +313,7 @@ void DataProcessor::ProcessAnImage(char* buf) {
 		RecordFirstIndices(fnum);
 	}
 
-	if (fileWriteEnable && *callbackAction == DO_EVERYTHING)
+	if (*fileWriteEnable && *callbackAction == DO_EVERYTHING)
 		file->WriteToFile(buf, generalData->fifoBufferSize + FILE_FRAME_HEADER_SIZE, fnum-firstMeasurementIndex);
 }
 

slsReceiverSoftware/src/HDF5File.cpp

@@ -26,6 +26,9 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
 		dataspace(0),
 		dataset(0),
 		datatype(PredType::STD_U16LE),
+		nPixelsX(nx),
+		nPixelsY(ny),
+		numFramesInFile(0),
 
 		dataspace_para(0),
 
@@ -35,11 +38,7 @@ HDF5File::HDF5File(int ind, int* nd, char* fname, char* fpath, uint64_t* findex,
 
 		para2("bunch_id"),
 		dataset_para2(0),
-		datatype_para2(PredType::STD_U64LE),
+		datatype_para2(PredType::STD_U64LE)
-
-		nPixelsX(nx),
-		nPixelsY(ny),
-		numFramesInFile(0)
 {
 #ifdef VERBOSE
 	PrintMembers();
@@ -65,29 +64,27 @@ void HDF5File::PrintMembers() {
 	}
 }
 
+
 void HDF5File::SetNumberofPixels(uint32_t nx, uint32_t ny) {
 	nPixelsX = nx;
 	nPixelsY = ny;
 }
 
+
 slsReceiverDefs::fileFormat HDF5File::GetFileType() {
 	return HDF5;
 }
 
+
 void HDF5File::UpdateDataType() {
 	switch(*dynamicRange){
-	case 16:
+	case 16:	datatype = PredType::STD_U16LE;		break;
-		datatype = PredType::STD_U16LE;
+	case 32:	datatype = PredType::STD_U32LE;		break;
-		break;
+	default:	datatype = PredType::STD_U8LE;		break;
-	case 32:
-		datatype = PredType::STD_U32LE;
-		break;
-	default:
-		datatype = PredType::STD_U8LE;
-		break;
 	}
 }
 
+
 int HDF5File::CreateFile(uint64_t fnum) {
 	numFramesInFile = 0;
 	currentFileName = HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
@@ -110,23 +107,17 @@ int HDF5File::CreateFile(uint64_t fnum) {
 	}
 	pthread_mutex_unlock(&Mutex);
 	printf("%d HDF5 File: %s\n", index, currentFileName.c_str());
-
-	//virtual file
-	if (master && (*detIndex==0))
-		return CreateVirtualFile(fnum);
-
 	return OK;
 }
 
+
 void HDF5File::CloseCurrentFile() {
 	pthread_mutex_lock(&Mutex);
 	HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
-	if (master && (*detIndex==0)) {
-		HDF5FileStatic::CloseVirtualDataFile(virtualfd);
-	}
 	pthread_mutex_unlock(&Mutex);
 }
 
+
 void HDF5File::CloseAllFiles() {
 	pthread_mutex_lock(&Mutex);
 	HDF5FileStatic::CloseDataFile(index, filefd, dataspace, dataset, dataset_para1, dataset_para2);
@@ -139,7 +130,7 @@ void HDF5File::CloseAllFiles() {
 
 
 int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
-	if (numFramesInFile >= maxFramesPerFile) {/**max  multiply by 100?????????????*/
+	if (numFramesInFile >= maxFramesPerFile) {
 		CloseCurrentFile();
 		CreateFile(fnum);
 	}
@@ -168,6 +159,7 @@ int HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum) {
 int HDF5File::CreateMasterFile(bool en, uint32_t size,
 		uint32_t nx, uint32_t ny, uint64_t at, uint64_t ap) {
 	if (master && (*detIndex==0)) {
+		virtualfd = 0;
 		masterFileName = HDF5FileStatic::CreateMasterFileName(filePath, fileNamePrefix, *fileIndex);
 		printf("Master File: %s\n", masterFileName.c_str());
 		pthread_mutex_lock(&Mutex);
@@ -180,77 +172,29 @@ int HDF5File::CreateMasterFile(bool en, uint32_t size,
 }
 
 
-int HDF5File::CreateVirtualFile(uint64_t fnum) {
+void HDF5File::EndofAcquisition(uint64_t numf) {
+	//not created before
+	if (!virtualfd)
+		CreateVirtualFile(numf);
+}
+
+
+int HDF5File::CreateVirtualFile(uint64_t numf) {
 	if (master && (*detIndex==0)) {
 
-		//file name
-		string virtualFileName = HDF5FileStatic::CreateVirtualFileName(filePath, fileNamePrefix, *fileIndex, *frameIndexEnable, fnum);
-		printf("Virtual File: %s\n", virtualFileName.c_str());
-
-		//source file names
-		int numReadouts = numDetX * numDetY;
-		string fileNames[numReadouts];
-		for (int i = 0; i < numReadouts; ++i) {
-			fileNames[i] = 	HDF5FileStatic::CreateFileName(filePath, fileNamePrefix, *fileIndex,
-					*frameIndexEnable, fnum, *detIndex, *numUnitsPerDetector, i);
-#ifdef VERBOSE
-			printf("%d: File Name: %s\n", i, fileNames[i].c_str());
-#endif
-		}
-
-		//datatype
-		hid_t cdatatype;
-		switch(*dynamicRange){
-		case 16:
-			cdatatype = H5T_STD_U16LE;
-			break;
-		case 32:
-			cdatatype = H5T_STD_U32LE;
-			break;
-		default:
-			cdatatype = H5T_STD_U8LE;
-			break;
-		}
-
-		//source dataset name
-		ostringstream osfn;
-		osfn << "/data";
-		if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
-		string srcDatasetName = osfn.str();
-
-		//virtual dataset name
-		osfn.str(""); osfn.clear();
-		osfn << "/virtual_data";
-		if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
-		string virtualDatasetName = osfn.str();
-		//parameter 1 name
-		osfn.str(""); osfn.clear();
-		osfn << "/virtual_" << para1;
-		if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
-		string vpara1DatasetName = osfn.str();
-
-		//parameter 2 name
-		osfn.str(""); osfn.clear();
-		osfn << "/virtual_" << para2;
-		if (*frameIndexEnable) osfn << "_f" << setfill('0') << setw(12) << fnum;
-		string vpara2DatasetName = osfn.str();
-
-		uint64_t framestosave = ((*numImages - fnum) > maxFramesPerFile) ? maxFramesPerFile : (*numImages-fnum);
-
-		//create virtual file
 		pthread_mutex_lock(&Mutex);
-		int ret = HDF5FileStatic::CreateVirtualDataFile(virtualfd, virtualFileName, virtualDatasetName, srcDatasetName,
+		int ret = HDF5FileStatic::CreateVirtualDataFile(
-				para1, vpara1DatasetName, H5T_STD_U32LE,
+				virtualfd, masterFileName,
-				para2, vpara2DatasetName, H5T_STD_U64LE,
+				filePath, fileNamePrefix, *fileIndex, *frameIndexEnable,
-				numReadouts, fileNames, *overWriteEnable, cdatatype,
+				*detIndex, *numUnitsPerDetector,
-				framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
+				maxFramesPerFile, numf,
-				framestosave, numDetY * nPixelsY, numDetX * ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX), HDF5_WRITER_VERSION);
+				"data", para1, para2,
-
+				datatype, datatype_para1, datatype_para2,
-		if (ret == OK)
+				numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
-			ret = HDF5FileStatic::LinkVirtualInMaster(masterFileName, virtualFileName, virtualDatasetName, vpara1DatasetName, vpara2DatasetName);
+				HDF5_WRITER_VERSION);
-
 		pthread_mutex_unlock(&Mutex);
 		return ret;
 	}
 	return OK;
 }
+

slsReceiverSoftware/src/UDPStandardImplementation.cpp

@@ -536,6 +536,17 @@ void UDPStandardImplementation::stopReceiver(){
 	while(DataProcessor::GetRunningMask()){
 		usleep(5000);
 	}
+
+	//create virtual file
+	if (fileWriteEnable && fileFormatType == HDF5) {
+		uint64_t maxFramescaught = 0;
+		for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
+			maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
+		}
+		if (maxFramescaught)
+			dataProcessor[0]->EndofAcquisition(maxFramescaught);
+	}
+
 	while(DataStreamer::GetRunningMask()){
 		usleep(5000);
 	}
@@ -637,8 +648,13 @@ void UDPStandardImplementation::shutDownUDPSockets() {
 
 
 void UDPStandardImplementation::closeFiles() {
-	for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it)
+	uint64_t maxFramescaught = 0;
+	for (vector<DataProcessor*>::const_iterator it = dataProcessor.begin(); it != dataProcessor.end(); ++it) {
 		(*it)->CloseFiles();
+		maxFramescaught = max(maxFramescaught, (*it)->GetNumFramesCaught());
+	}
+	if (maxFramescaught)
+		dataProcessor[0]->EndofAcquisition(maxFramescaught);
 }