WIP

slsReceiverSoftware/src/HDF5File.cpp

@@ -94,7 +94,7 @@ HDF5File::~HDF5File() {
 
 void HDF5File::PrintMembers(TLogLevel level) {
 	File::PrintMembers();
-	UpdateDataType();
+	UpdateDataTypeFromDr();
 	if (datatype == PredType::STD_U8LE) {
 		LOG(level) << "Data Type: 4 or 8";
 	} else if (datatype == PredType::STD_U16LE) {
@@ -118,7 +118,7 @@ slsDetectorDefs::fileFormat HDF5File::GetFileType() {
 }
 
 
-void HDF5File::UpdateDataType() {
+void HDF5File::UpdateDataTypeFromDr() {
 	switch(*dynamicRange){
 	case 16:	datatype = PredType::STD_U16LE;		break;
 	case 32:	datatype = PredType::STD_U32LE;		break;
@@ -131,36 +131,26 @@ void HDF5File::CreateFile() {
 	numFilesinAcquisition++;
 	numFramesInFile = 0;
 	numActualPacketsInFile = 0;
-	currentFileName = HDF5FileStatic::CreateFileName(*filePath, *fileNamePrefix, *fileIndex,
-			subFileIndex, *detIndex, *numUnitsPerDetector, index);
+
+	std::ostringstream os;
+	os << *filePath << "/" << *fileNamePrefix << "_d"
+		<< (*detIndex * (*numUnitsPerDetector) + index) << "_f" << subFileIndex << '_'
+		<< *fileIndex << ".h5";
+	currentFileName = os.str();
 
 	//first time
 	if(subFileIndex == 0u) 
-		UpdateDataType();
+		UpdateDataTypeFromDr();
 
-	uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
-			(((extNumImages - subFileIndex) > (*maxFramesPerFile)) ?  // save up to maximum at a time
-					(*maxFramesPerFile) : (extNumImages-subFileIndex)));
-
-	std::lock_guard<std::mutex> lock(mutex);
-	HDF5FileStatic::CreateDataFile(index, *overWriteEnable, currentFileName, (*numImages > 1),
-			subFileIndex, framestosave, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
-			datatype, filefd, dataspace, dataset,
-			HDF5_WRITER_VERSION, MAX_CHUNKED_IMAGES,
-			dataspace_para,	dataset_para,
-			parameterNames, parameterDataTypes);
+	CreateDataFile();
 
 	if(!(*silentMode)) {
 		LOG(logINFO) << *udpPortNumber << ": HDF5 File created: " << currentFileName;
 	}
 }
 
-
 void HDF5File::CloseCurrentFile() {
-	{
-		std::lock_guard<std::mutex> lock(mutex);
-		HDF5FileStatic::CloseDataFile(index, filefd);
-	}
+	CloseFile(filefd, nullptr, false);
 	for (unsigned int i = 0; i < dataset_para.size(); ++i)
 		delete dataset_para[i];
 	dataset_para.clear();
@@ -174,11 +164,10 @@ void HDF5File::CloseCurrentFile() {
 void HDF5File::CloseAllFiles() {
 	numFilesinAcquisition = 0;
 	{
-		std::lock_guard<std::mutex> lock(mutex);
-		HDF5FileStatic::CloseDataFile(index, filefd);
+		CloseFile(filefd, nullptr, false);
 		if (master && (*detIndex==0)) {
-			HDF5FileStatic::CloseMasterDataFile(masterfd);
-			HDF5FileStatic::CloseVirtualDataFile(virtualfd);
+			CloseFile(masterfd, nullptr, true);
+			CloseFile(nullptr, &virtualfd, false);
 		}
 	}
 	for (unsigned int i = 0; i < dataset_para.size(); ++i)
@@ -191,7 +180,7 @@ void HDF5File::CloseAllFiles() {
 }
 
 
-void HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t nump) {
+void HDF5File::WriteToFile(char* buffer, int bufferSize, uint64_t currentFrameNumber, uint32_t numPacketsCaught) {
 
 	// check if maxframesperfile = 0 for infinite
 	if ((*maxFramesPerFile) && (numFramesInFile >= (*maxFramesPerFile))) {
@@ -200,58 +189,33 @@ void HDF5File::WriteToFile(char* buffer, int buffersize, uint64_t fnum, uint32_t
 		CreateFile();
 	}
 	numFramesInFile++;
-	numActualPacketsInFile += nump;
-
-	std::lock_guard<std::mutex> lock(mutex);
+	numActualPacketsInFile += numPacketsCaught;
 
 	// extend dataset (when receiver start followed by many status starts (jungfrau)))
-	if (fnum >= extNumImages) {
-		HDF5FileStatic::ExtendDataset(index, dataspace, dataset,
-				dataspace_para, dataset_para, *numImages);
-		if (!(*silentMode)) {
-			LOG(logINFO) << index << " Extending HDF5 dataset by " <<
-					extNumImages << ", Total x Dimension: " << (extNumImages + *numImages);
-		}
-		extNumImages += *numImages;
+	if (currentFrameNumber >= extNumImages) {
+		ExtendDataset();
 	}
 
-	HDF5FileStatic::WriteDataFile(index, buffer + sizeof(sls_receiver_header),
-			// infinite then no need for %maxframesperfile
-			((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)),
-			nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
-			dataspace, dataset, datatype);
-
-	HDF5FileStatic::WriteParameterDatasets(index, dataspace_para,
-				// infinite then no need for %maxframesperfile
-				((*maxFramesPerFile == 0) ? fnum : fnum%(*maxFramesPerFile)),
-				dataset_para, (sls_receiver_header*) (buffer),
-				parameterDataTypes);
+	WriteDataFile(currentFrameNumber, buffer + sizeof(sls_receiver_header));
+	WriteParameterDatasets(currentFrameNumber, (sls_receiver_header*) (buffer));
 }
 
 
-void HDF5File::CreateMasterFile(bool mfwenable, masterAttributes& attr) {
+void HDF5File::CreateMasterFile(bool masterFileWriteEnable, masterAttributes& masterFileAttributes) {
 
 	//beginning of every acquisition
 	numFramesInFile = 0;
 	numActualPacketsInFile = 0;
 	extNumImages = *numImages;
 
-	if (mfwenable && master && (*detIndex==0)) {
+	if (masterFileWriteEnable && master && (*detIndex==0)) {
 		virtualfd = 0;
-		masterFileName = HDF5FileStatic::CreateMasterFileName(*filePath,
-				*fileNamePrefix, *fileIndex);
-		if(!(*silentMode)) {
-			LOG(logINFO) << "Master File: " << masterFileName;
-		}
-		std::lock_guard<std::mutex> lock(mutex);
-		attr.version = HDF5_WRITER_VERSION;
-		HDF5FileStatic::CreateMasterDataFile(masterfd, masterFileName,
-				*overWriteEnable, attr);
+		CreateMasterDataFile();
 	}
 }
 
 
-void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
+void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numImagesCaught) {
 	//not created before
 	if (!virtualfd && anyPacketsCaught) {
 
@@ -260,11 +224,11 @@ void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
 
 			//only one file and one sub image (link current file in master)
 			if (((numFilesinAcquisition == 1) && (numDetY*numDetX) == 1)) {
-				LinkVirtualFileinMasterFile();
+				LinkVirtualInMaster();
 			}
 			//create virutal file
 			else{
-				CreateVirtualFile(numf);}
+				CreateVirtualFile(numImagesCaught);}
 		}
 	}
 	numFilesinAcquisition = 0;
@@ -272,35 +236,684 @@ void HDF5File::EndofAcquisition(bool anyPacketsCaught, uint64_t numf) {
 
 
 // called only by the one maser receiver
-void HDF5File::CreateVirtualFile(uint64_t numf) {
+void HDF5File::CreateVirtualFile(uint64_t numImagesCaught) {
+
+	HDF5FileStatic::CreateVirtualDataFile(
+		// infinite images in 1 file, then maxfrperfile = numImagesCaught
+		((*maxFramesPerFile == 0) ? numImagesCaught+1 : *maxFramesPerFile),
+		numImagesCaught + 1);
+}
+
+void HDF5File::CloseFile(H5File* fd, hid_t* cfd, bool master) {
 	std::lock_guard<std::mutex> lock(mutex);
 
-	std::string vname = HDF5FileStatic::CreateVirtualFileName(*filePath, *fileNamePrefix, *fileIndex);
+	// c code due to only c implementation of H5Pset_virtual available
+	if (cfd != nullptr) {
+			if(*cfd) {
+				if (H5Fclose(*cfd) < 0 ) {
+					LOG(logERROR) << "Could not close virtual HDF5 handles";
+				}
+				*cfd = 0;
+			}
+	} else {
+		Exception::dontPrint(); // to handle errors
+			if (fd) {
+				delete fd;
+				fd = 0;
+			}
+		} catch(const Exception& error) {
+			LOG(logERROR) << "Could not close " 
+			<< master ? "master" : "data" 
+			<< " HDF5 handles of index " << index;
+			error.printErrorStack();
+		}
+	}
+}
+
+void HDF5File::WriteDataFile(uint64_t currentFrameNumber, char* buffer) {
+	std::lock_guard<std::mutex> lock(mutex);
+
+	uint64_t nDimx = ((*maxFramesPerFile == 0) ? 
+		currentFrameNumber : currentFrameNumber % (*maxFramesPerFile));
+	uint32_t nDimy = nPixelsY;
+	uint32_t nDimz = ((*dynamicRange == 4) ? (nPixelsX / 2) : nPixelsX);
+
+	hsize_t count[3] = {1, nDimy, nDimz};
+	hsize_t start[3] = {nDimx, 0, 0};
+	hsize_t dims2[2] = {nDimy, nDimz};
+	try{
+		Exception::dontPrint(); //to handle errors
+
+		dataspace->selectHyperslab( H5S_SELECT_SET, count, start);
+		DataSpace memspace(2,dims2);
+		dataset->write(buffer, datatype, memspace, *dataspace);
+		memspace.close();
+	}
+	catch(const Exception& error){
+		LOG(logERROR) << "Could not write to file in object " << index;
+		error.printErrorStack();
+		throw RuntimeError("Could not write to file in object " + 
+			std::to_string(index));
+	}
+}
+
+void HDF5File::WriteParameterDatasets(uint64_t currentFrameNumber, 
+	sls_receiver_header* rheader) {
+	std::lock_guard<std::mutex> lock(mutex);
+
+	uint64_t fnum = ((*maxFramesPerFile == 0) ? 
+		currentFrameNumber : currentFrameNumber % (*maxFramesPerFile));
+
+	sls_detector_header header = rheader->detHeader;
+	hsize_t count[1] = {1};
+	hsize_t start[1] = {fnum};
+	int i = 0;
+	try{
+		Exception::dontPrint(); //to handle errors
+		dataspace_para->selectHyperslab( H5S_SELECT_SET, count, start);
+		DataSpace memspace(H5S_SCALAR);
+		dataset_para[0]->write(&header.frameNumber, 	
+			parameterDataTypes[0], memspace, *dataspace_para);i=1;
+		dataset_para[1]->write(&header.expLength, 		
+			parameterDataTypes[1], memspace, *dataspace_para);i=2;
+		dataset_para[2]->write(&header.packetNumber, 	
+			parameterDataTypes[2], memspace, *dataspace_para);i=3;
+		dataset_para[3]->write(&header.bunchId, 		
+			parameterDataTypes[3], memspace, *dataspace_para);i=4;
+		dataset_para[4]->write(&header.timestamp, 		
+			parameterDataTypes[4], memspace, *dataspace_para);i=5;
+		dataset_para[5]->write(&header.modId, 			
+			parameterDataTypes[5], memspace, *dataspace_para);i=6;
+		dataset_para[6]->write(&header.row, 			
+			parameterDataTypes[6], memspace, *dataspace_para);i=7;
+		dataset_para[7]->write(&header.column, 		
+			parameterDataTypes[7], memspace, *dataspace_para);i=8;
+		dataset_para[8]->write(&header.reserved, 		
+			parameterDataTypes[8], memspace, *dataspace_para);i=9;
+		dataset_para[9]->write(&header.debug, 			
+			parameterDataTypes[9], memspace, *dataspace_para);i=10;
+		dataset_para[10]->write(&header.roundRNumber, 	
+			parameterDataTypes[10], memspace, *dataspace_para);i=11;
+		dataset_para[11]->write(&header.detType, 		
+			parameterDataTypes[11], memspace, *dataspace_para);i=12;
+		dataset_para[12]->write(&header.version, 		
+			parameterDataTypes[12], memspace, *dataspace_para);i=13;
+
+		// contiguous bitset
+		if (sizeof(sls_bitset) == sizeof(bitset_storage)) {
+			dataset_para[13]->write((char*)&(rheader->packetsMask), 
+				parameterDataTypes[13], memspace, *dataspace_para);
+		}
+
+		// not contiguous bitset
+		else {
+			// get contiguous representation of bit mask
+			bitset_storage storage;
+			memset(storage, 0 , sizeof(bitset_storage));
+			sls_bitset bits = rheader->packetsMask;
+			for (int i = 0; i < MAX_NUM_PACKETS; ++i)
+				storage[i >> 3] |= (bits[i] << (i & 7));
+			// write bitmask
+			dataset_para[13]->write((char*)storage, parameterDataTypes[13], 
+				memspace, *dataspace_para);
+		}i=14;
+	}
+	catch(const Exception& error){
+		error.printErrorStack();
+		throw RuntimeError("Could not write parameters (index:" + 
+			std::to_string(i) + ") to file in object " + std::to_string(index));
+	}
+}
+
+void HDF5File::ExtendDataset() {
+	std::lock_guard<std::mutex> lock(mutex);
+
+	try{
+		Exception::dontPrint(); //to handle errors
+
+		hsize_t dims[3];
+		dataspace->getSimpleExtentDims(dims);
+		dims[0] += *numImages;
+
+		dataset->extend(dims);
+		delete dataspace;
+		dataspace = 0;
+		dataspace = new DataSpace(dataset->getSpace());
+
+		hsize_t dims_para[1] = {dims[0]};
+		for (unsigned int i = 0; i < dataset_para.size(); ++i)
+			dataset_para[i]->extend(dims_para);
+		delete dataspace_para; 
+		dataspace_para = 0;
+		dspace_dataspace_parapara = new DataSpace(dataset_para[0]->getSpace());
+
+	}
+	catch(const Exception& error){
+		error.printErrorStack();
+		throw RuntimeError("Could not extend dataset in object " + 
+			std::to_string(index));
+	}
+	if (!(*silentMode)) {
+		LOG(logINFO) << index << " Extending HDF5 dataset by " <<
+				extNumImages << ", Total x Dimension: " << (extNumImages + *numImages);
+	}
+	extNumImages += *numImages;
+}
+
+void HDF5File::CreateMasterDataFile() {
+	std::lock_guard<std::mutex> lock(mutex);
+	
+	std::ostringstream os;
+	os << *filePath << "/" << *fileNamePrefix << "_master"
+		<< "_" << *fileIndex << ".h5";
+	masterFileName = os.str();
+
+	if(!(*silentMode)) {
+		LOG(logINFO) << "Master File: " << masterFileName;
+	}
+	masterFileAttributes.version = HDF5_WRITER_VERSION;
+
+	try {
+		Exception::dontPrint(); //to handle errors
+
+		FileAccPropList flist;
+		flist.setFcloseDegree(H5F_CLOSE_STRONG);
+		masterfd = 0;
+		if(!(*overWriteEnable))
+			masterfd = new H5File( masterFileName.c_str(), H5F_ACC_EXCL,
+					FileCreatPropList::DEFAULT,
+					flist );
+		else
+			masterfd = new H5File( masterFileName.c_str(), H5F_ACC_TRUNC,
+					FileCreatPropList::DEFAULT,
+					flist );
+
+		//variables
+		DataSpace dataspace = DataSpace (H5S_SCALAR);
+		Attribute attribute;
+		DataSet dataset;
+		int iValue=0;
+		double dValue=0;
+		StrType strdatatype(PredType::C_S1,256);
+
+		//create attributes
+		//version
+		dValue = masterFileAttributes.version;
+		attribute = masterfd->createAttribute("version",PredType::NATIVE_DOUBLE, dataspace);
+		attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+		//Create a group in the file
+		Group group1( masterfd->createGroup( "entry" ) );
+			Group group2( group1.createGroup("data") );
+			Group group3( group1.createGroup("instrument") );
+			Group group4( group3.createGroup("beam") );
+			Group group5( group3.createGroup("detector") );
+			Group group6( group1.createGroup("sample") );
+
+		//Dynamic Range
+		dataset = group5.createDataSet ( "dynamic range", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.dynamicRange), PredType::NATIVE_INT);
+		attribute = dataset.createAttribute("unit",strdatatype, dataspace);
+		attribute.write(strdatatype, std::string("bits"));
+
+		//Ten Giga
+		iValue = masterFileAttributes.tenGiga;
+		dataset = group5.createDataSet ( "ten giga enable", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &iValue, PredType::NATIVE_INT);
+
+		//Image Size
+		dataset = group5.createDataSet ( "image size", PredType::NATIVE_INT, dataspace );
+		dataset.write (  &(masterFileAttributes.imageSize), PredType::NATIVE_INT);
+		attribute = dataset.createAttribute("unit",strdatatype, dataspace);
+		attribute.write(strdatatype, std::string("bytes"));
+
+		//x
+		dataset = group5.createDataSet ( "number of pixels in x axis", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.nPixelsX), PredType::NATIVE_INT);
+
+		//y
+		dataset = group5.createDataSet ( "number of pixels in y axis", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.nPixelsY), PredType::NATIVE_INT);
+
+		//Maximum frames per file
+		dataset = group5.createDataSet ( "maximum frames per file", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.maxFramesPerFile), PredType::NATIVE_INT);
+
+		//Total Frames
+		dataset = group5.createDataSet ( "total frames", PredType::STD_U64LE, dataspace );
+		dataset.write ( &(masterFileAttributes.totalFrames), PredType::STD_U64LE);
+
+		//Exptime
+		dataset = group5.createDataSet ( "exposure time", PredType::STD_U64LE, dataspace );
+		dataset.write ( &(masterFileAttributes.exptimeNs), PredType::STD_U64LE);
+		attribute = dataset.createAttribute("unit",strdatatype, dataspace);
+		attribute.write(strdatatype, std::string("ns"));
+
+		//SubExptime
+		dataset = group5.createDataSet ( "sub exposure time", PredType::STD_U64LE, dataspace );
+		dataset.write ( &(masterFileAttributes.subExptimeNs), PredType::STD_U64LE);
+		attribute = dataset.createAttribute("unit",strdatatype, dataspace);
+		attribute.write(strdatatype, std::string("ns"));
+
+		//SubPeriod
+		dataset = group5.createDataSet ( "sub period", PredType::STD_U64LE, dataspace );
+		dataset.write ( &(masterFileAttributes.subPeriodNs), PredType::STD_U64LE);
+		attribute = dataset.createAttribute("unit",strdatatype, dataspace);
+		attribute.write(strdatatype, std::string("ns"));
+
+		//Period
+		dataset = group5.createDataSet ( "acquisition period", PredType::STD_U64LE, dataspace );
+		dataset.write ( &(masterFileAttributes.periodNs), PredType::STD_U64LE);
+		attribute = dataset.createAttribute("unit",strdatatype, dataspace);
+		attribute.write(strdatatype, std::string("ns"));
+
+		//Quad Enable
+		dataset = group5.createDataSet ( "quad enable", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.quadEnable), PredType::NATIVE_INT);
+
+		//Analog Flag
+		dataset = group5.createDataSet ( "analog flag", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.analogFlag), PredType::NATIVE_INT);
+
+		//Digital Flag
+		dataset = group5.createDataSet ( "digital flag", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.digitalFlag), PredType::NATIVE_INT);
+
+		//ADC Mask
+		dataset = group5.createDataSet ( "adc mask", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.adcmask), PredType::NATIVE_INT);
+
+		//Dbit Offset
+		dataset = group5.createDataSet ( "dbit offset", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.dbitoffset), PredType::NATIVE_INT);
+
+		// Dbit List
+		dataset = group5.createDataSet ( "dbit bitset list", PredType::STD_U64LE, dataspace );
+		dataset.write ( &(masterFileAttributes.dbitlist), PredType::STD_U64LE);
+
+		// Roi xmin
+		dataset = group5.createDataSet ( "roi xmin", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.roiXmin), PredType::NATIVE_INT);
+
+		// Roi xmax
+		dataset = group5.createDataSet ( "roi xmax", PredType::NATIVE_INT, dataspace );
+		dataset.write ( &(masterFileAttributes.roiXmax), PredType::NATIVE_INT);
+
+		//Timestamp
+		time_t t = time(0);
+		dataset = group5.createDataSet ( "timestamp", strdatatype, dataspace );
+		dataset.write ( std::string(ctime(&t)), strdatatype );
+
+		masterfd->close();
+
+	} catch(const Exception& error) {
+		error.printErrorStack();
+		if (masterfd) masterfd->close();
+		throw RuntimeError("Could not create master HDF5 handles");
+	}
+}
+
+void HDF5File::CreateDataFile() {
+	std::lock_guard<std::mutex> lock(mutex);
+	
+	uint64_t framestosave = ((*maxFramesPerFile == 0) ? *numImages : // infinite images
+		(((extNumImages - subFileIndex) > (*maxFramesPerFile)) ?  // save up to maximum at a time
+				(*maxFramesPerFile) : (extNumImages-subFileIndex)));
+
+	uint64_t nDimx = framestosave;
+	uint32_t nDimy = nPixelsY;
+	uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
+
+	try {
+		Exception::dontPrint(); //to handle errors
+
+		//file
+		FileAccPropList fapl;
+		fapl.setFcloseDegree(H5F_CLOSE_STRONG);
+		filefd = 0;
+		if(!(*overWriteEnable))
+			filefd = new H5File( currentFileName.c_str(), H5F_ACC_EXCL,
+					FileCreatPropList::DEFAULT,
+					fapl );
+		else
+			filefd = new H5File( currentFileName.c_str(), H5F_ACC_TRUNC,
+					FileCreatPropList::DEFAULT,
+					fapl );
+
+		//attributes - version
+		double dValue = HDF5_WRITER_VERSION;
+		DataSpace dataspace_attr = DataSpace (H5S_SCALAR);
+		Attribute attribute = filefd->createAttribute("version",
+			PredType::NATIVE_DOUBLE, dataspace_attr);
+		attribute.write(PredType::NATIVE_DOUBLE, &dValue);
+
+		//dataspace
+		hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
+		hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
+		dataspace = 0;
+		dataspace = new DataSpace (3,srcdims,srcdimsmax);
+
+
+		//dataset name
+		std::ostringstream osfn;
+		osfn << "/data";
+		if (*numImages > 1) 
+			osfn << "_f" << std::setfill('0') << std::setw(12) << subFileIndex;
+		std::string dsetname = osfn.str();
+
+		//dataset
+		//fill value
+		DSetCreatPropList plist;
+		int fill_value = -1;
+		plist.setFillValue(datatype, &fill_value);
+		// always create chunked dataset as unlimited is only 
+		// supported with chunked layout
+		hsize_t chunk_dims[3] = {MAX_CHUNKED_IMAGES, nDimy, nDimz};
+		plist.setChunk(3, chunk_dims);
+		dataset = 0;
+		dataset = new DataSet (filefd->createDataSet(dsetname.c_str(), datatype, 
+			*dataspace, plist));
+
+		//create parameter datasets
+		hsize_t dims[1] = {nDimx};
+		hsize_t dimsmax[1] = {H5S_UNLIMITED};
+		dataspace_para = 0;
+		dataspace_para = new DataSpace (1,dims,dimsmax);
+
+		// always create chunked dataset as unlimited is only 
+		// supported with chunked layout
+		DSetCreatPropList paralist;
+		hsize_t chunkpara_dims[3] = {MAX_CHUNKED_IMAGES};
+		paralist.setChunk(1, chunkpara_dims);
+
+		for (unsigned int i = 0; i < parameterNames.size(); ++i){
+			DataSet* ds = new DataSet(filefd->createDataSet(parameterNames[i],
+					parameterDataTypes[i], *dataspace_para, paralist));
+			dataset_para.push_back(ds);
+		}
+	}
+	catch(const Exception& error){
+		error.printErrorStack();
+		if (filefd) filefd->close();
+		throw RuntimeError("Could not create HDF5 handles in object " + index);
+	}
+}
+
+void HDF5File::CreateVirtualDataFile(uint32_t maxFramesPerFile, uint64_t numf) {
+	std::lock_guard<std::mutex> lock(mutex);
+
+	std::ostringstream osfn;
+	osfn << *filePath << "/" << *fileNamePrefix;
+	osfn << "_virtual";
+	osfn << "_" << *fileIndex;
+	osfn << ".h5";
+	std::string vname =  osfn.str();
+
 	if(!(*silentMode)) {
 		LOG(logINFO) << "Virtual File: " << vname;
 	}
-	HDF5FileStatic::CreateVirtualDataFile(vname,
-			virtualfd, masterFileName,
-			*filePath, *fileNamePrefix, *fileIndex, (*numImages > 1),
-			*detIndex, *numUnitsPerDetector,
-			// infinite images in 1 file, then maxfrperfile = numf
-			((*maxFramesPerFile == 0) ? numf+1 : *maxFramesPerFile),
-			numf+1,
-			"data",	datatype,
-			numDetY, numDetX, nPixelsY, ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX),
-			HDF5_WRITER_VERSION,
-			parameterNames, parameterDataTypes);
+
+	int numDetz = numDetX;
+	uint32_t nDimy = nPixelsY;
+	uint32_t nDimz = ((*dynamicRange==4) ? (nPixelsX/2) : nPixelsX);
+
+	try {
+		//file
+		hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
+		if (dfal < 0)
+			throw RuntimeError("Could not create file access property for virtual file " + vname);
+		if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
+			throw RuntimeError("Could not set strong file close degree for virtual file " + vname);
+		virtualfd = H5Fcreate( vname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, dfal);
+		if (virtualfd < 0)
+			throw RuntimeError("Could not create virtual file " + vname);
+
+		//attributes - version
+		hid_t dataspace_attr = H5Screate (H5S_SCALAR);
+		if (dataspace_attr < 0)
+			throw RuntimeError("Could not create dataspace for attribute in virtual file " + vname);
+		hid_t attrid = H5Acreate2 (virtualfd, "version", H5T_NATIVE_DOUBLE, dataspace_attr, H5P_DEFAULT, H5P_DEFAULT);
+		if (attrid < 0)
+			throw RuntimeError("Could not create attribute in virtual file " + vname);
+		double attr_data = HDF5_WRITER_VERSION;
+		if (H5Awrite (attrid, H5T_NATIVE_DOUBLE, &attr_data) < 0)
+			throw RuntimeError("Could not write attribute in virtual file " + vname);
+		if (H5Aclose (attrid) < 0)
+			throw RuntimeError("Could not close attribute in virtual file " + vname);
+
+
+		//virtual dataspace
+		hsize_t vdsdims[3] = {numf, numDetY * nDimy, numDetz * nDimz};
+		hid_t vdsDataspace = H5Screate_simple(3, vdsdims ,NULL);
+		if (vdsDataspace < 0)
+			throw RuntimeError("Could not create virtual dataspace in virtual file " + vname);
+		hsize_t vdsdims_para[2] = {numf, (unsigned int) numDetY * numDetz};
+		hid_t vdsDataspace_para = H5Screate_simple(2, vdsdims_para, NULL);
+		if (vdsDataspace_para < 0)
+			throw RuntimeError("Could not create virtual dataspace (parameters) in virtual file " + vname);
+
+
+		//fill values
+		hid_t dcpl = H5Pcreate (H5P_DATASET_CREATE);
+		if (dcpl < 0)
+			throw RuntimeError("Could not create file creation properties in virtual file " + vname);
+		int fill_value = -1;
+		if (H5Pset_fill_value (dcpl, GetDataTypeinC(datatype), &fill_value) < 0)
+			throw RuntimeError("Could not create fill value in virtual file " + vname);
+		hid_t dcpl_para[parameterNames.size()];
+		for (unsigned int i = 0; i < parameterNames.size(); ++i) {
+			dcpl_para[i] = H5Pcreate (H5P_DATASET_CREATE);
+			if (dcpl_para[i] < 0)
+				throw RuntimeError("Could not create file creation properties (parameters) in virtual file " + vname);
+			if (H5Pset_fill_value (dcpl_para[i], GetDataTypeinC(parameterDataTypes[i]), &fill_value) < 0)
+				throw RuntimeError("Could not create fill value (parameters) in virtual file " + vname);
+		}
+
+		//hyperslab
+		int numMajorHyperslab = numf/maxFramesPerFile;
+		if (numf%maxFramesPerFile) numMajorHyperslab++;
+		uint64_t framesSaved = 0;
+		for (int j = 0; j < numMajorHyperslab; j++) {
+
+			uint64_t nDimx = ((numf - framesSaved) > maxFramesPerFile)
+					? maxFramesPerFile : (numf-framesSaved);
+			hsize_t offset[3] 		= {framesSaved, 0, 0};
+			hsize_t	count[3] 		= {nDimx, nDimy, nDimz};
+			hsize_t offset_para[2] 	= {framesSaved, 0};
+			hsize_t	count_para[2] 	= {nDimx, 1};
+
+			for (int i = 0; i < numDetY * numDetz; ++i) {
+
+				//setect hyperslabs
+				if (H5Sselect_hyperslab (vdsDataspace, H5S_SELECT_SET, offset, NULL, count, NULL) < 0) {
+					throw RuntimeError("Could not select hyperslab");
+				}
+				if (H5Sselect_hyperslab (vdsDataspace_para, H5S_SELECT_SET,
+						offset_para, NULL, count_para, NULL) < 0) {
+					throw RuntimeError("Could not select hyperslab for parameters");
+				}
+
+				//source file name
+				std::ostringstream os;
+				os << *filePath << "/" << *fileNamePrefix << "_d"
+					<< (*detIndex * (*numUnitsPerDetector) + i) << "_f" << j << '_'
+					<< *fileIndex << ".h5";
+				std::string srcFileName =  os.str();
+
+				LOG(logERROR) << srcFileName;
+				// find relative path
+				std::string relative_srcFileName = srcFileName;
+				{
+					size_t i = srcFileName.rfind('/', srcFileName.length());
+					if (i != std::string::npos)
+						relative_srcFileName = (srcFileName.substr(i+1, srcFileName.length() - i));
+				}
+
+				//source dataset name
+				std::ostringstream osfn;
+				osfn << "/data";
+				if (*numImages > 1) 
+					osfn << "_f" << std::setfill('0') << std::setw(12) << j;
+				std::string srcDatasetName = osfn.str();
+
+				//source dataspace
+				hsize_t srcdims[3] = {nDimx, nDimy, nDimz};
+				hsize_t srcdimsmax[3] = {H5S_UNLIMITED, nDimy, nDimz};
+				hid_t srcDataspace = H5Screate_simple(3, srcdims, srcdimsmax);
+				if (srcDataspace < 0)
+					throw RuntimeError("Could not create source dataspace in virtual file " + vname);
+				hsize_t srcdims_para[1] = {nDimx};
+				hsize_t srcdimsmax_para[1] = {H5S_UNLIMITED};
+				hid_t srcDataspace_para = H5Screate_simple(1, srcdims_para, srcdimsmax_para);
+				if (srcDataspace_para < 0)
+					throw RuntimeError("Could not create source dataspace (parameters) in virtual file " + vname);
+
+				//mapping
+				if (H5Pset_virtual(dcpl, vdsDataspace, relative_srcFileName.c_str(),
+						srcDatasetName.c_str(), srcDataspace) < 0) {
+					throw RuntimeError("Could not set mapping for paramter 1");
+				}
+
+				for (unsigned int k = 0; k < parameterNames.size(); ++k) {
+					if (H5Pset_virtual(dcpl_para[k], vdsDataspace_para, relative_srcFileName.c_str(),
+							parameterNames[k], srcDataspace_para) < 0) {
+						throw RuntimeError("Could not set mapping for paramter " + std::to_string(k));
+					}
+				}
+
+				//H5Sclose(srcDataspace);
+				//H5Sclose(srcDataspace_para);
+				offset[2] += nDimz;
+				if (offset[2] >=  (numDetz * nDimz)) {
+					offset[2] = 0;
+					offset[1] += nDimy;
+				}
+				offset_para[1]++;
+			}
+			framesSaved += nDimx;
+		}
+
+		//dataset
+		std::string virtualDatasetName = "data";
+		hid_t vdsdataset = H5Dcreate2 (virtualfd, virtualDatasetName.c_str(),
+				GetDataTypeinC(datatype), vdsDataspace, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+		if (vdsdataset < 0)
+			throw RuntimeError("Could not create virutal dataset in virtual file " + vname);
+
+
+		//virtual parameter dataset
+		for (unsigned int i = 0; i < parameterNames.size(); ++i) {
+			hid_t vdsdataset_para = H5Dcreate2 (virtualfd,
+					parameterNames[i],
+					GetDataTypeinC(parameterDataTypes[i]), vdsDataspace_para,
+					H5P_DEFAULT, dcpl_para[i], H5P_DEFAULT);
+			if (vdsdataset_para < 0)
+				throw RuntimeError("Could not create virutal dataset (parameters) in virtual file " + vname);
+		}
+
+		//close
+		H5Fclose(virtualfd); 
+		virtualfd = 0;
+
+		//link
+		LinkVirtualInMaster(masterFileName, vname, virtualDatasetName, parameterNames);
+	} catch (const RuntimeError &e) {
+		if (virtualfd > 0)
+			H5Fclose(virtualfd);
+		virtualfd = 0;
+	}
 }
 
-// called only by the one maser receiver
-void HDF5File::LinkVirtualFileinMasterFile() {
+void HDF5File::LinkVirtualInMaster() {
+	std::lock_guard<std::mutex> lock(mutex);
+
 	//dataset name
 	std::ostringstream osfn;
 	osfn << "/data";
 	if ((*numImages > 1)) osfn << "_f" << std::setfill('0') << std::setw(12) << 0;
 	std::string dsetname = osfn.str();
 
-	std::lock_guard<std::mutex> lock(mutex);
-	HDF5FileStatic::LinkVirtualInMaster(masterFileName, currentFileName,
-			dsetname, parameterNames);
+	char linkname[100];
+	hid_t vfd = 0;
+
+	try {
+		hid_t dfal = H5Pcreate (H5P_FILE_ACCESS);
+		if (dfal < 0)
+			throw RuntimeError("Could not create file access property for link");
+		if (H5Pset_fclose_degree (dfal, H5F_CLOSE_STRONG) < 0)
+			throw RuntimeError("Could not set strong file close degree for link");
+
+		//open master file
+		hid_t mfd = H5Fopen( masterFileName.c_str(), H5F_ACC_RDWR, dfal);
+		if (mfd < 0)
+			throw RuntimeError("Could not open master file");
+
+		//open virtual file
+		vfd = H5Fopen( currentFileName.c_str(), H5F_ACC_RDWR, dfal);
+		if (vfd < 0) {
+			H5Fclose(mfd); mfd = 0;
+			throw RuntimeError("Could not open virtual file");
+		}
+
+		// find relative path
+		std::string relative_virtualfname = currentFileName;
+		{
+			size_t i = currentFileName.rfind('/', currentFileName.length());
+			if (i != std::string::npos)
+				relative_virtualfname = (currentFileName.substr(i+1, currentFileName.length() - i));
+		}
+
+		//**data dataset**
+		hid_t vdset = H5Dopen2( vfd, dsetname.c_str(), H5P_DEFAULT);
+		if (vdset < 0) {
+			H5Fclose(mfd);
+			throw RuntimeError("Could not open virtual data dataset");
+		}
+		sprintf(linkname, "/entry/data/%s",dsetname.c_str());
+		if(H5Lcreate_external( relative_virtualfname.c_str(), dsetname.c_str(),
+										mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
+			H5Fclose(mfd); mfd = 0;
+			throw RuntimeError("Could not create link to data dataset");
+		}
+		H5Dclose(vdset);
+
+		//**paramter datasets**
+		for (unsigned int i = 0; i < parameterNames.size(); ++i){
+			hid_t vdset_para = H5Dopen2( vfd, (std::string (parameterNames[i])).c_str(), H5P_DEFAULT);
+			if (vdset_para < 0) {
+				H5Fclose(mfd); mfd = 0;
+				throw RuntimeError("Could not open virtual parameter dataset to create link");
+			}
+			sprintf(linkname, "/entry/data/%s",(std::string (parameterNames[i])).c_str());
+
+			if(H5Lcreate_external( relative_virtualfname.c_str(), (std::string (parameterNames[i])).c_str(),
+					mfd, linkname, H5P_DEFAULT, H5P_DEFAULT) < 0) {
+				H5Fclose(mfd); mfd = 0;
+				throw RuntimeError("Could not create link to virtual parameter dataset");
+			}
+		}
+
+		H5Fclose(mfd); mfd = 0;
+		H5Fclose(vfd); vfd = 0;
+	} catch () {
+		if(vfd > 0)
+			H5Fclose(vfd);
+		vfd = 0;
+	}
 }
+
+hid_t HDF5File::GetDataTypeinC(DataType dtype) {
+	if (dtype == PredType::STD_U8LE)
+		return H5T_STD_U8LE;
+	else if (dtype == PredType::STD_U16LE)
+		return H5T_STD_U16LE;
+	else if (dtype == PredType::STD_U32LE)
+		return H5T_STD_U32LE;
+	else if (dtype == PredType::STD_U64LE)
+		return H5T_STD_U64LE;
+	else {
+		hid_t s = H5Tcopy(H5T_C_S1);
+		H5Tset_size(s, MAX_NUM_PACKETS);
+		return s;
+	}
+}