The GraphicsMagick plugin is limited to 8 and 16-bit integer arrays. It supports @@ -167,39 +168,39 @@
The GraphicsMagick plugin supports the Int32 parameter NDFileMagickCompressType to control the compression mode of the file. NDFileMagick.template defines 2 records - to support this: $(P)$(R)CompressMode (longout) and $(P)$(R)CompressMode_RBV (longin). - The following are the supported compression modes:
+ to support this: $(P)$(R)CompressType (longout) and $(P)$(R)CompressType_RBV (longin). + The following are the supported compression types:- No formats support all of these compression modes. Many support only one, or have - an implicit compression mode and so ignore the CompressMode parameter. For example, + No formats support all of these compression types. Many support only one, or have + an implicit compression mode and so ignore the CompressType parameter. For example, the JPEG format supports only the JPEG compression scheme, which is implicit. The - TIFF format supports "None", ...
+ TIFF format supports "None", (ADD DOCUMENTATION ON TIFF COMPRESSION AND OTHER FORMATS)- The NDFileNetTIFF class + The NDFileMagick class documentation describes this class in detail.
- The NDFileTIFF plugin is created with the NDFileTIFFConfigure command, either from + The NDFileMagick plugin is created with the NDFileMagickConfigure command, either from C/C++ or from the EPICS IOC shell.
-NDFileTIFFConfigure (const char *portName, int queueSize, int blockingCallbacks, - const char *NDArrayPort, int NDArrayAddr, size_t maxMemory, - int priority, int stackSize) +NDFileMagickConfigure(const char *portName, int queueSize, int blockingCallbacks, + const char *NDArrayPort, int NDArrayAddr, size_t maxMemory, + int priority, int stackSize)For details on the meaning of the parameters to this function refer to the detailed - documentation on the NDFileTIFFConfigure function in the - NDFileTIFF.cpp documentation and in the documentation for the constructor - for the NDFileTIFF class. + documentation on the NDFileMagickConfigure function in the + NDFileMagick.cpp documentation and in the documentation for the constructor + for the NDFileMagick class.
netCDF file plugin @@ -278,54 +279,54 @@ variables: :dimOffset = 0, 0, 0 ; :dimBinning = 1, 2, 2 ; :dimReverse = 0, 0, 0 ; - :Attr_colorMode_DataType = "Int32" ; - :Attr_colorMode_Description = "Color mode" ; + :Attr_colorMode_DataType = "Int32" ; + :Attr_colorMode_Description = "Color mode" ; :Attr_colorMode_Source = ; - :Attr_colorMode_SourceType = "Driver" ; - :Attr_AcquireTime_DataType = "Float64" ; - :Attr_AcquireTime_Description = "Camera acquire time" ; - :Attr_AcquireTime_Source = "13SIM1:cam1:AcquireTime" ; - :Attr_AcquireTime_SourceType = "EPICS_PV" ; - :Attr_RingCurrent_DataType = "Float64" ; - :Attr_RingCurrent_Description = "Storage ring current" ; - :Attr_RingCurrent_Source = "S:SRcurrentAI" ; - :Attr_RingCurrent_SourceType = "EPICS_PV" ; - :Attr_ImageCounter_DataType = "Int32" ; - :Attr_ImageCounter_Description = "Image counter" ; - :Attr_ImageCounter_Source = "ARRAY_COUNTER" ; - :Attr_ImageCounter_SourceType = "Param" ; - :Attr_CameraModel_DataType = "String" ; - :Attr_CameraModel_Description = "Camera model" ; - :Attr_CameraModel_Source = "MODEL" ; - :Attr_CameraModel_SourceType = "Param" ; - :Attr_BinX_DataType = "Int32" ; - :Attr_BinX_Description = "X binning" ; - :Attr_BinX_Source = "13SIM1:ROI1:0:BinX_RBV" ; - :Attr_BinX_SourceType = "EPICS_PV" ; - :Attr_BinY_DataType = "Int32" ; - :Attr_BinY_Description = "Y binning" ; - :Attr_BinY_Source = "13SIM1:ROI1:0:BinY_RBV" ; - :Attr_BinY_SourceType = "EPICS_PV" ; - :Attr_AttrTimeStamp_DataType = "Float64" ; - :Attr_AttrTimeStamp_Description = "Time stamp" ; - :Attr_AttrTimeStamp_Source = "TIME_STAMP" ; - :Attr_AttrTimeStamp_SourceType = "Param" ; - :Attr_ROI0Mean_DataType = "Float64" ; - :Attr_ROI0Mean_Description = "Mean value ROI 0" ; - :Attr_ROI0Mean_Source = "MEAN_VALUE" ; - :Attr_ROI0Mean_SourceType = "Param" ; - :Attr_ROI1Mean_DataType = "Float64" ; - :Attr_ROI1Mean_Description = "Mean value ROI 0" ; - :Attr_ROI1Mean_Source = "MEAN_VALUE" ; - :Attr_ROI1Mean_SourceType = "Param" ; - :Attr_FilePath_DataType = "String" ; - :Attr_FilePath_Description = "File path" ; - :Attr_FilePath_Source = "13SIM1:netCDF1:FilePath_RBV" ; - :Attr_FilePath_SourceType = "EPICS_PV" ; - :Attr_FileName_DataType = "String" ; - :Attr_FileName_Description = "File name" ; - :Attr_FileName_Source = "13SIM1:netCDF1:FileName_RBV" ; - :Attr_FileName_SourceType = "EPICS_PV" ; + :Attr_colorMode_SourceType = "Driver" ; + :Attr_AcquireTime_DataType = "Float64" ; + :Attr_AcquireTime_Description = "Camera acquire time" ; + :Attr_AcquireTime_Source = "13SIM1:cam1:AcquireTime" ; + :Attr_AcquireTime_SourceType = "EPICS_PV" ; + :Attr_RingCurrent_DataType = "Float64" ; + :Attr_RingCurrent_Description = "Storage ring current" ; + :Attr_RingCurrent_Source = "S:SRcurrentAI" ; + :Attr_RingCurrent_SourceType = "EPICS_PV" ; + :Attr_ImageCounter_DataType = "Int32" ; + :Attr_ImageCounter_Description = "Image counter" ; + :Attr_ImageCounter_Source = "ARRAY_COUNTER" ; + :Attr_ImageCounter_SourceType = "Param" ; + :Attr_CameraModel_DataType = "String" ; + :Attr_CameraModel_Description = "Camera model" ; + :Attr_CameraModel_Source = "MODEL" ; + :Attr_CameraModel_SourceType = "Param" ; + :Attr_BinX_DataType = "Int32" ; + :Attr_BinX_Description = "X binning" ; + :Attr_BinX_Source = "13SIM1:ROI1:0:BinX_RBV" ; + :Attr_BinX_SourceType = "EPICS_PV" ; + :Attr_BinY_DataType = "Int32" ; + :Attr_BinY_Description = "Y binning" ; + :Attr_BinY_Source = "13SIM1:ROI1:0:BinY_RBV" ; + :Attr_BinY_SourceType = "EPICS_PV" ; + :Attr_AttrTimeStamp_DataType = "Float64" ; + :Attr_AttrTimeStamp_Description = "Time stamp" ; + :Attr_AttrTimeStamp_Source = "TIME_STAMP" ; + :Attr_AttrTimeStamp_SourceType = "Param" ; + :Attr_ROI0Mean_DataType = "Float64" ; + :Attr_ROI0Mean_Description = "Mean value ROI 0" ; + :Attr_ROI0Mean_Source = "MEAN_VALUE" ; + :Attr_ROI0Mean_SourceType = "Param" ; + :Attr_ROI1Mean_DataType = "Float64" ; + :Attr_ROI1Mean_Description = "Mean value ROI 0" ; + :Attr_ROI1Mean_Source = "MEAN_VALUE" ; + :Attr_ROI1Mean_SourceType = "Param" ; + :Attr_FilePath_DataType = "String" ; + :Attr_FilePath_Description = "File path" ; + :Attr_FilePath_Source = "13SIM1:netCDF1:FilePath_RBV" ; + :Attr_FilePath_SourceType = "EPICS_PV" ; + :Attr_FileName_DataType = "String" ; + :Attr_FileName_Description = "File name" ; + :Attr_FileName_Source = "13SIM1:netCDF1:FileName_RBV" ; + :Attr_FileName_SourceType = "EPICS_PV" ; }
ncdump is one of a number of very useful command line utilities that come with the @@ -471,6 +472,486 @@ variables: on older Linux systems (which predate Fedora Core 8 for example), it was decided to use this older version of HDF5. Future releases of areaDetector may use HDF5 1.8.2 or later, and hence not work with older Linux systems.
++ NDFileHDF5 inherits from NDPluginFile. This plugin uses the HDF5 libraries to store + data. HDF5 file format is a self-describing binary format supported by the + hdfgroup. +
++ The plugin supports all NDArray datatypes and any number of NDArray dimensions (tested + up to 3). It supports storing multiple NDArrays in a single file (in stream or capture + modes) where each NDArray get appended to an extra dimension. +
++ NDArray attributes are stored in the HDF5 file. In case of multi-frame files the + attributes are stored in 1D datasets (arrays). +
++ The NDFileHDF5 plugin is created with the NDFileHDF5Configure command, either from + C/C++ or from the EPICS IOC shell.
++NDFileHDF5Configure (const char *portName, int queueSize, int blockingCallbacks, + const char *NDArrayPort, int NDArrayAddr, size_t maxMemory, + int priority, int stackSize) ++
+ For details on the meaning of the parameters to this function refer to the detailed + documentation on the NDFileNexusConfigure function in the + NDFileHDF5.cpp documentation and in the documentation for the constructor + for the NDFileHDF5 class.
++ The HDF5 files comprises a hierachial data structure, similar to a file system structure + with directories (groups) and files (datasets) [ref] +
++ This plugin writes HDF5 files that are compatible with the Nexus file format. This + is achieved by defining a specific hierachial structure of groups and datasets and + by tagging elements in the hierachi with certain "NX_class=nnn" attributes. + Although Nexus libraries are not used to write the data to disk, this file structure + allow Nexus-aware readers to open and read the content of these HDF5 files. This + has been tested with the Nexus reader in the GDA application. +
++ The directory structure of the HDF5 files, generated by this plugin:
++entry <-- NX_class=NXentry +| ++--instrument <-- NX_class=NXinstrument + | + +--NDAttributes + | | + | +--- <-- Any number of EPICS PV based NDAttributes as individual 1D datasets + | +--- The PV attributes are in the 'instrument' group as they are expected + | to originate from the beamline/instrument rather than the detector itself + | + +--detector <-- NX_class=NXdetector + | | + | +--data <-- NX_class=SDS, signal=1 + | | + | +--NDAttributes + | | + | +--- <-- Any number of PARAM type NDAttributes as individual 1D datasets + | +--- These type of parameters are in the 'detector' group as they + | originate from the areaDetector system + | + +--performance <-- Performance of the file writing + | + +--timestamp <-- A 2D dataset of different timing measurements taking during file writing ++
+ HDFView is + a simple GUI tool for viewing and browsing HDF files. It has some limited support + for viewing images, plotting graphs and displaying data tables. +
++ The HDF5 libraries + also ships with a number of command-line tools for browsing and dumping data. +
++ The screenshot below shows the hdfview application with a datafile open. The datafile + is generated by the plugin and a number of elements are visible:
++ Both areaDetector and the HDF5 file format supports multidimensional datasets. The + dimensions from the NDArray are preserved when writing to the HDF5 file. In multi-frame + files (plugin in Stream or Capture mode) an additional dimension is added to the + dataset to hold the array of frames. +
++ In addition to the dimensions of the NDArray it is also possible to specify up to + 2 extra "virtual" dimensions to store datasets in the file. This is to + support applications where a sample is scanned in up to two dimensions, say X and + Y. For each scan point a dataset comprising of multiple frames can be stored. The + length of (i.e. number of points in) each of the two virtual dimensions have to + be specified before the plugin opens the file for writing. This feature is only + supported in the Stream and Capture modes. +
++ This feature allow for creating very large sets of scan data which matches the dimensions + of the performed scan in one datafile. Depending on the application this can be + a benefit in post processing. +
++ The figure below illustrate the use of the two extra "virtual" dimensions + in a 2D (X,Y) raster scan with N frames per point:
+
+ Prior to starting a scan like this the user will need to configure the number of + virtual dimensions to use (none, 1 or 2); the number of frames per point; and the + length of each of the virtual dimensions (4 x 2 in the example figure). It is not + possible to change the number or size of dimensions while the file is open. +
++ For 2D image (greyscale) formats the dimensions in the multiframe HDF5 file are + organised as follows:
++ The attributes from NDArrays are stored in the HDF5 files. The list of attributes + is loaded when a file is opened so XML attributes files should not be reloaded during + an acquisition run. +
++ The type of attribute is used to determine where in the file structure the attribute + data will end up. All attribute datasets will be named by the NDArray attribute + name. They will also have metadata (known as HDF "attributes") to indicate + their source type and origin.
++ The HDF5 library supports a number of compression algorithms. When using HDF5 libraries + to write and read files the compression is seemless: it only need to be switched + on when writing and HDF5 enabled applications can read the files without any additional + configuration. Only one compression filter can be applied at the time. +
++ The following compression filters are supported in the NDFileHDF5 plugin:
+| + Parameter Definitions and EPICS Record Definitions in NDFileHDF5.template | +||||||
| + Parameter index variable | ++ asyn interface | ++ Access | ++ Description | ++ drvInfo string | ++ EPICS record name | ++ EPICS record type | +
|---|---|---|---|---|---|---|
| + nRowChunks | ++ asynInt32 | ++ r/w | ++ Configure HDF5 "chunking" to approriate size for the filesystem: sets + number of rows to use per chunk | ++ HDF5_nRowChunks | +
+ $(P)$(R)NumRowChunks + $(P)$(R)NumRowChunks_RBV |
+
+ longout + longin |
+
| + storeAttributes | ++ asynInt32 | ++ r/w | ++ Enable or disable support for storing NDArray attributes in file | ++ HDF5_storeAttributes | +
+ $(P)$(R)StoreAttr + $(P)$(R)StoreAttr_RBV |
+
+ bo + bi |
+
| + storePerformance | ++ asynInt32 | ++ r/w | ++ Enable or disable support for storing file IO timing measurements in file | ++ HDF5_storePerformance | +
+ $(P)$(R)StorePerform + $(P)$(R)StorePerform_RBV |
+
+ bo + bi |
+
| + Additional Virtual Dimensions | +||||||
| + nExtraDims | ++ asynInt32 | ++ r/w | ++ Number of extra dimensions [0..2] | ++ HDF5_nExtraDims | +
+ $(P)$(R)NumExtraDims + $(P)$(R)NumExtraDims |
+
+ mbbo + mbbi |
+
| + extraDimSizeN | ++ asynInt32 | ++ r/w | ++ Size of extra dimension N (no. of frames per point) | ++ HDF5_extraDimSizeN | +
+ $(P)$(R)ExtraDimSizeN + $(P)$(R)ExtraDimSizeN_RBV |
+ + | +
| + extraDimSizeX | ++ asynInt32 | ++ r/w | ++ Size of extra dimension X | ++ HDF5_extraDimSizeX | +
+ $(P)$(R)ExtraDimSizeX + $(P)$(R)ExtraDimSizeX_RBV |
+
+ longout + longin |
+
| + extraDimSizeY | ++ asynInt32 | ++ r/w | ++ Size of extra dimension Y | ++ HDF5_extraDimSizeY | +
+ $(P)$(R)ExtraDimSizeY + $(P)$(R)ExtraDimSizeY_RBV |
+
+ longout + longin |
+
| + Runtime Statistics | +||||||
| + totalRuntime | ++ asynFloat64 | ++ r/o | ++ Total runtime in seconds from first frame to file closed | ++ HDF5_totalRuntime | ++ $(P)$(R)Runtime | ++ ai | +
| + totalIoSpeed | ++ asynFloat64 | ++ r/o | ++ Overall IO write speed in megabit per second from first frame to file closed | ++ HDF5_totalIoSpeed | ++ $(P)$(R)IOSpeed | ++ ai | +
| + Compression Filters | +||||||
| + compressionType | ++ asynInt32 | ++ r/w | ++ Select or switch off compression filter | ++ HDF5_compressionType | +
+ $(P)$(R)Compression + $(P)$(R)Compression_RBV |
+
+ mbbo + mbbi |
+
| + nbitsPrecision | ++ asynInt32 | ++ r/w | ++ N-bit compression filter: number of data bits per pixel | ++ HDF5_nbitsPrecision | +
+ $(P)$(R)NumDataBits + $(P)$(R)NumDataBits_RBV |
+
+ longout + longin |
+
| + nbitsOffset | ++ asynInt32 | ++ r/w | ++ N-bit compression filter: dataword bit-offset in pixel | ++ HDF5_nbitsOffset | +
+ $(P)$(R)DataBitsOffset + $(P)$(R)DataBitsOffset_RBV |
+
+ longout + longin |
+
| + szipNumPixels | ++ asynInt32 | ++ r/w | ++ szip compression filter: number of pixels in filter [1..32] | ++ HDF5_szipNumPixels | +
+ $(P)$(R)SZipNumPixels + $(P)$(R)SZipNumPixels_RBV |
+
+ longout + longin |
+
| + zCompressLevel | ++ asynInt32 | ++ r/w | ++ zlib compression filter: compression level [1..9] | ++ HDF5_zCompressLevel | +
+ $(P)$(R)ZLevel + $(P)$(R)ZLevel_RBV |
+
+ longout + longin |
+
+ The NDFileHDF5 plugin was developed by Ulrik Kofoed Pedersen at Diamond Light Source.