Update README.md

This makes "make" build only the most useful software by default.

Building the test_plugin target requires fortran and openmp, which are
otherwise not required for the durin plugin itself, and is not of
interest to most building the software.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2018, Diamond Light Source
+Copyright (c) 2018-2019, Diamond Light Source
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -8,15 +8,15 @@ modification, are permitted provided that the following conditions are met:
     * Redistributions in binary form must reproduce the above copyright
       notice, this list of conditions and the following disclaimer in the
       documentation and/or other materials provided with the distribution.
-    * Neither the name of the <organization> nor the
+    * Neither the name of the copyright holder nor the
       names of its contributors may be used to endorse or promote products
       derived from this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

Makefile

@@ -3,15 +3,19 @@ SRC_DIR = ./src
 TEST_DIR = ./test
 INC_DIR = $(SRC_DIR)
 
-CC=h5cc
+BSLZ4_SRC_DIR = ./bslz4/src
-CFLAGS=-Wall -g -O2 -fpic -I$(INC_DIR)
+BSLZ4_BUILD_DIR = ./bslz4/build
+BSLZ4_INC_DIR = $(BSLZ4_SRC_DIR)
 
-.PHONY: all
+CC=h5cc
-all: plugin example test_plugin
+CFLAGS=-Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=c89
 
 .PHONY: plugin
 plugin: $(BUILD_DIR)/durin-plugin.so
 
+.PHONY: all
+all: plugin example test_plugin
+
 .PHONY: example
 example: $(BUILD_DIR)/example
 
@@ -26,14 +30,26 @@ $(BUILD_DIR)/%.o: $(SRC_DIR)/%.c
 	mkdir -p $(BUILD_DIR)
 	$(CC) $(CFLAGS) -c $< -o $@
 
-$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o
+$(BSLZ4_BUILD_DIR)/%.o: $(BSLZ4_SRC_DIR)/%.c
+	mkdir -p $(BSLZ4_BUILD_DIR)
+	$(CC) $(CFLAGS) -c $< -o $@
+
+$(BUILD_DIR)/bslz4.a: $(BSLZ4_BUILD_DIR)/lz4.o $(BSLZ4_BUILD_DIR)/bitshuffle.o \
+$(BSLZ4_BUILD_DIR)/bitshuffle_core.o $(BSLZ4_BUILD_DIR)/iochain.o
+	mkdir -p $(BUILD_DIR)
+	ar rcs $@ $^
+
+$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o \
+$(BUILD_DIR)/bslz4.a
 	mkdir -p $(BUILD_DIR)
 	$(CC) $(CFLAGS) -shared $^ -o $(BUILD_DIR)/durin-plugin.so
 
-$(BUILD_DIR)/example: $(BUILD_DIR)/test.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o
+$(BUILD_DIR)/example: $(BUILD_DIR)/test.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o \
+$(BUILD_DIR)/bslz4.a
 	mkdir -p $(BUILD_DIR)
 	$(CC) $(CFLAGS) $^ -o $(BUILD_DIR)/example
 
 .PHONY: clean
 clean:
 	rm -r $(BUILD_DIR)
+	rm -r $(BSLZ4_BUILD_DIR)

README.md

@@ -29,8 +29,69 @@ series of datasets named `data_000001`, `data_000002`, etc.
 
 
 ## Building
+
+### Building HDF5 library
+The HDF5 library used when building durin must have been compiled with specific switches enabled
+to allow the durin plugin to be built and used.
+
+Download the HDF5 source code (https://www.hdfgroup.org/downloads/hdf5/source-code) and extract
+to any directory (referred to as `/hdf5_dir`), and run the following commands.
 ```
-cd durin
+cd /hdf5_dir
+mkdir build
+cd build
+export CFLAGS=-fPIC
+../configure --enable-threadsafe --enable-deprecated-symbols --enable-hl --enable-unsupported
+make
+make check
+make install
+```
+The hdf5 tools and libraries should now be located in `/hdf5_dir/build/hdf5`
+
+For reference, the plugin requires the thread-safe switch and the optimised chunk read function.
+The chunk read function may be defined in the high level library instead of the regular library,
+depending on the exact HDF5 version downloaded (hence the --enable-deprecated-symbols _and_ --enable-hl).
+The unsupported flag enables building with both threadsafe and high-level enabled.
+
+
+### Building durin plugin
+The plugin makefile will use the "h5cc" compiler wrapper, provided by the HDF5 library, which
+must be on your PATH.
+Download or clone the plugin source code (https://github.com/DiamondLightSource/durin)
+into any directory (referred to as `/durin_dir`) and run the following commands.
+```
+cd /durin_dir
+PATH=/hdf5_dir/build/hdf5/bin:$PATH
 make
 ```
-Plugin file is `build/durin-plugin.so`
+The plugin is located at `/durin_dir/build/durin-plugin.so` and should be added to the
+XDS.INP file as `LIB=/durin_dir/build/durin-plugin.so`
+
+
+
+## Example XDS.INP
+```
+DETECTOR=PILATUS MINIMUM_VALID_PIXEL_VALUE=0 OVERLOAD=4096
+LIB=/opt/durin/build/durin-plugin.so
+SENSOR_THICKNESS= 0.450
+!SENSOR_MATERIAL / THICKNESS Si 0.450
+!SILICON= 3.953379
+DIRECTION_OF_DETECTOR_X-AXIS= 1.00000 0.00000 0.00000
+DIRECTION_OF_DETECTOR_Y-AXIS= 0.00000 1.00000 0.00000
+DETECTOR_DISTANCE= 194.633000
+ORGX= 1041.30 ORGY= 1160.90
+ROTATION_AXIS= 0.00000 -1.00000 -0.00000
+STARTING_ANGLE= -30.000
+OSCILLATION_RANGE= 0.100
+X-RAY_WAVELENGTH= 0.97891
+INCIDENT_BEAM_DIRECTION= -0.000 -0.000 1.022
+FRACTION_OF_POLARIZATION= 0.999
+POLARIZATION_PLANE_NORMAL= 0.000 1.000 0.000
+NAME_TEMPLATE_OF_DATA_FRAMES= ../image_9264_??????.h5
+TRUSTED_REGION= 0.0 1.41
+DATA_RANGE= 1 600
+JOB=XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT
+```
+
+N.B. the master file is needed, not the .nxs one which follows the
+standard.

bslz4/LICENSE

@@ -0,0 +1,21 @@
+Bitshuffle - Filter for improving compression of typed binary data.
+
+Copyright (c) 2014 Kiyoshi Masui (kiyo@physics.ubc.ca)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

bslz4/src/bitshuffle.c

@@ -0,0 +1,165 @@
+/*
+ * Bitshuffle - Filter for improving compression of typed binary data.
+ *
+ * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
+ * Website: http://www.github.com/kiyo-masui/bitshuffle
+ * Created: 2014
+ *
+ * See LICENSE file for details about copyright and rights to use.
+ *
+ */
+
+#include "bitshuffle.h"
+#include "bitshuffle_core.h"
+#include "bitshuffle_internals.h"
+#include "lz4.h"
+
+#include <stdio.h>
+#include <string.h>
+
+
+
+
+#define BSHUF_LZ4_DECOMPRESS_FAST
+
+
+
+#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) {                      \
+    free(buf); return count - 1000; }
+
+
+/* Bitshuffle and compress a single block. */
+int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \
+        const size_t size, const size_t elem_size) {
+
+    int64_t nbytes, count;
+    void *tmp_buf_bshuf;
+    void *tmp_buf_lz4;
+    size_t this_iter;
+    const void *in;
+    void *out;
+
+    tmp_buf_bshuf = malloc(size * elem_size);
+    if (tmp_buf_bshuf == NULL) return -1;
+
+    tmp_buf_lz4 = malloc(LZ4_compressBound(size * elem_size));
+    if (tmp_buf_lz4 == NULL){
+        free(tmp_buf_bshuf);
+        return -1;
+    }
+
+
+    in = ioc_get_in(C_ptr, &this_iter);
+    ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size));
+
+    count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size);
+    if (count < 0) {
+        free(tmp_buf_lz4);
+        free(tmp_buf_bshuf);
+        return count;
+    }
+    nbytes = LZ4_compress((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size);
+    free(tmp_buf_bshuf);
+    CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4);
+
+    out = ioc_get_out(C_ptr, &this_iter);
+    ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4));
+
+    bshuf_write_uint32_BE(out, nbytes);
+    memcpy((char *) out + 4, tmp_buf_lz4, nbytes);
+
+    free(tmp_buf_lz4);
+
+    return nbytes + 4;
+}
+
+
+/* Decompress and bitunshuffle a single block. */
+int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,
+        const size_t size, const size_t elem_size) {
+
+    int64_t nbytes, count;
+    void *out, *tmp_buf;
+    const void *in;
+    size_t this_iter;
+    int32_t nbytes_from_header;
+
+    in = ioc_get_in(C_ptr, &this_iter);
+    nbytes_from_header = bshuf_read_uint32_BE(in);
+    ioc_set_next_in(C_ptr, &this_iter,
+            (void*) ((char*) in + nbytes_from_header + 4));
+
+    out = ioc_get_out(C_ptr, &this_iter);
+    ioc_set_next_out(C_ptr, &this_iter,
+            (void *) ((char *) out + size * elem_size));
+
+    tmp_buf = malloc(size * elem_size);
+    if (tmp_buf == NULL) return -1;
+
+#ifdef BSHUF_LZ4_DECOMPRESS_FAST
+    nbytes = LZ4_decompress_fast((const char*) in + 4, (char*) tmp_buf, size * elem_size);
+    CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
+    if (nbytes != nbytes_from_header) {
+        free(tmp_buf);
+        return -91;
+    }
+#else
+    nbytes = LZ4_decompress_safe((const char*) in + 4, (char *) tmp_buf, nbytes_from_header,
+                                 size * elem_size);
+    CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
+    if (nbytes != size * elem_size) {
+        free(tmp_buf);
+        return -91;
+    }
+    nbytes = nbytes_from_header;
+#endif
+    count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size);
+    CHECK_ERR_FREE(count, tmp_buf);
+    nbytes += 4;
+
+    free(tmp_buf);
+    return nbytes;
+}
+
+
+/* ---- Public functions ----
+ *
+ * See header file for description and usage.
+ *
+ */
+
+size_t bshuf_compress_lz4_bound(const size_t size,
+        const size_t elem_size, size_t block_size) {
+
+    size_t bound, leftover;
+
+    if (block_size == 0) {
+        block_size = bshuf_default_block_size(elem_size);
+    }
+    if (block_size % BSHUF_BLOCKED_MULT) return -81;
+
+
+
+    bound = (LZ4_compressBound(block_size * elem_size) + 4) * (size / block_size);
+
+    leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
+    if (leftover) bound += LZ4_compressBound(leftover * elem_size) + 4;
+
+    bound += (size % BSHUF_BLOCKED_MULT) * elem_size;
+    return bound;
+}
+
+
+int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size,
+        const size_t elem_size, size_t block_size) {
+    return bshuf_blocked_wrap_fun(&bshuf_compress_lz4_block, in, out, size,
+            elem_size, block_size);
+}
+
+
+int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
+        const size_t elem_size, size_t block_size) {
+    return bshuf_blocked_wrap_fun(&bshuf_decompress_lz4_block, in, out, size,
+            elem_size, block_size);
+}
+

bslz4/src/bitshuffle.h

@@ -0,0 +1,123 @@
+/*
+ * Bitshuffle - Filter for improving compression of typed binary data.
+ *
+ * This file is part of Bitshuffle
+ * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
+ * Website: http://www.github.com/kiyo-masui/bitshuffle
+ * Created: 2014
+ *
+ * See LICENSE file for details about copyright and rights to use.
+ *
+ *
+ * Header File
+ *
+ * Worker routines return an int64_t which is the number of bytes processed
+ * if positive or an error code if negative.
+ *
+ * Error codes:
+ *      -1    : Failed to allocate memory.
+ *      -11   : Missing SSE.
+ *      -12   : Missing AVX.
+ *      -80   : Input size not a multiple of 8.
+ *      -81   : block_size not multiple of 8.
+ *      -91   : Decompression error, wrong number of bytes processed.
+ *      -1YYY : Error internal to compression routine with error code -YYY.
+ */
+
+
+#ifndef BITSHUFFLE_H
+#define BITSHUFFLE_H
+
+#include <stdlib.h>
+#include "bitshuffle_core.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ---- bshuf_compress_lz4_bound ----
+ *
+ * Bound on size of data compressed with *bshuf_compress_lz4*.
+ *
+ * Parameters
+ * ----------
+ *  size : number of elements in input
+ *  elem_size : element size of typed data
+ *  block_size : Process in blocks of this many elements. Pass 0 to
+ *  select automatically (recommended).
+ *
+ * Returns
+ * -------
+ *  Bound on compressed data size.
+ *
+ */
+size_t bshuf_compress_lz4_bound(const size_t size,
+        const size_t elem_size, size_t block_size);
+
+
+/* ---- bshuf_compress_lz4 ----
+ *
+ * Bitshuffled and compress the data using LZ4.
+ *
+ * Transpose within elements, in blocks of data of *block_size* elements then
+ * compress the blocks using LZ4.  In the output buffer, each block is prefixed
+ * by a 4 byte integer giving the compressed size of that block.
+ *
+ * Output buffer must be large enough to hold the compressed data.  This could
+ * be in principle substantially larger than the input buffer.  Use the routine
+ * *bshuf_compress_lz4_bound* to get an upper limit.
+ *
+ * Parameters
+ * ----------
+ *  in : input buffer, must be of size * elem_size bytes
+ *  out : output buffer, must be large enough to hold data.
+ *  size : number of elements in input
+ *  elem_size : element size of typed data
+ *  block_size : Process in blocks of this many elements. Pass 0 to
+ *  select automatically (recommended).
+ *
+ * Returns
+ * -------
+ *  number of bytes used in output buffer, negative error-code if failed.
+ *
+ */
+int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, const size_t
+        elem_size, size_t block_size);
+
+
+/* ---- bshuf_decompress_lz4 ----
+ *
+ * Undo compression and bitshuffling.
+ *
+ * Decompress data then un-bitshuffle it in blocks of *block_size* elements.
+ *
+ * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
+ * must patch the parameters used to compress the data.
+ *
+ * NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function
+ * LZ4_decompress_fast from LZ4, which does not protect against maliciously
+ * formed datasets. By modifying the compressed data, this function could be
+ * coerced into leaving the boundaries of the input buffer.
+ *
+ * Parameters
+ * ----------
+ *  in : input buffer
+ *  out : output buffer, must be of size * elem_size bytes
+ *  size : number of elements in input
+ *  elem_size : element size of typed data
+ *  block_size : Process in blocks of this many elements. Pass 0 to
+ *  select automatically (recommended).
+ *
+ * Returns
+ * -------
+ *  number of bytes consumed in *input* buffer, negative error-code if failed.
+ *
+ */
+int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
+        const size_t elem_size, size_t block_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

bslz4/src/bitshuffle_core.h

@@ -0,0 +1,157 @@
+/*
+ * Bitshuffle - Filter for improving compression of typed binary data.
+ *
+ * This file is part of Bitshuffle
+ * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
+ * Website: http://www.github.com/kiyo-masui/bitshuffle
+ * Created: 2014
+ *
+ * See LICENSE file for details about copyright and rights to use.
+ *
+ *
+ * Header File
+ *
+ * Worker routines return an int64_t which is the number of bytes processed
+ * if positive or an error code if negative.
+ *
+ * Error codes:
+ *      -1    : Failed to allocate memory.
+ *      -11   : Missing SSE.
+ *      -12   : Missing AVX.
+ *      -80   : Input size not a multiple of 8.
+ *      -81   : block_size not multiple of 8.
+ *      -91   : Decompression error, wrong number of bytes processed.
+ *      -1YYY : Error internal to compression routine with error code -YYY.
+ */
+
+
+#ifndef BITSHUFFLE_CORE_H
+#define BITSHUFFLE_CORE_H
+
+
+#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
+#include <stdint.h>
+#else
+  typedef unsigned char       uint8_t;
+  typedef unsigned short      uint16_t;
+  typedef signed short        int16_t;
+  typedef unsigned int        uint32_t;
+  typedef signed int          int32_t;
+  typedef unsigned long long  uint64_t;
+  typedef long long           int64_t;
+#endif
+
+#include <stdlib.h>
+
+
+
+#ifndef BSHUF_VERSION_MAJOR
+#define BSHUF_VERSION_MAJOR 0
+#define BSHUF_VERSION_MINOR 3
+#define BSHUF_VERSION_POINT 4
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* --- bshuf_using_SSE2 ----
+ *
+ * Whether routines where compiled with the SSE2 instruction set.
+ *
+ * Returns
+ * -------
+ *  1 if using SSE2, 0 otherwise.
+ *
+ */
+int bshuf_using_SSE2(void);
+
+
+/* ---- bshuf_using_AVX2 ----
+ *
+ * Whether routines where compiled with the AVX2 instruction set.
+ *
+ * Returns
+ * -------
+ *  1 if using AVX2, 0 otherwise.
+ *
+ */
+int bshuf_using_AVX2(void);
+
+
+/* ---- bshuf_default_block_size ----
+ *
+ * The default block size as function of element size.
+ *
+ * This is the block size used by the blocked routines (any routine
+ * taking a *block_size* argument) when the block_size is not provided
+ * (zero is passed).
+ *
+ * The results of this routine are guaranteed to be stable such that
+ * shuffled/compressed data can always be decompressed.
+ *
+ * Parameters
+ * ----------
+ *  elem_size : element size of data to be shuffled/compressed.
+ *
+ */
+size_t bshuf_default_block_size(const size_t elem_size);
+
+
+/* ---- bshuf_bitshuffle ----
+ *
+ * Bitshuffle the data.
+ *
+ * Transpose the bits within elements, in blocks of *block_size*
+ * elements.
+ *
+ * Parameters
+ * ----------
+ *  in : input buffer, must be of size * elem_size bytes
+ *  out : output buffer, must be of size * elem_size bytes
+ *  size : number of elements in input
+ *  elem_size : element size of typed data
+ *  block_size : Do transpose in blocks of this many elements. Pass 0 to
+ *  select automatically (recommended).
+ *
+ * Returns
+ * -------
+ *  number of bytes processed, negative error-code if failed.
+ *
+ */
+int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size,
+        const size_t elem_size, size_t block_size);
+
+
+/* ---- bshuf_bitunshuffle ----
+ *
+ * Unshuffle bitshuffled data.
+ *
+ * Untranspose the bits within elements, in blocks of *block_size*
+ * elements.
+ *
+ * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
+ * must match the parameters used to shuffle the data.
+ *
+ * Parameters
+ * ----------
+ *  in : input buffer, must be of size * elem_size bytes
+ *  out : output buffer, must be of size * elem_size bytes
+ *  size : number of elements in input
+ *  elem_size : element size of typed data
+ *  block_size : Do transpose in blocks of this many elements. Pass 0 to
+ *  select automatically (recommended).
+ *
+ * Returns
+ * -------
+ *  number of bytes processed, negative error-code if failed.
+ *
+ */
+int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size,
+        const size_t elem_size, size_t block_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

bslz4/src/bitshuffle_internals.h

@@ -0,0 +1,63 @@
+/*
+ * Bitshuffle - Filter for improving compression of typed binary data.
+ *
+ * This file is part of Bitshuffle
+ * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
+ * Website: http://www.github.com/kiyo-masui/bitshuffle
+ * Created: 2014
+ *
+ * See LICENSE file for details about copyright and rights to use.
+ */
+
+
+#ifndef BITSHUFFLE_INTERNALS_H
+#define BITSHUFFLE_INTERNALS_H
+
+
+#include <stdlib.h>
+#include "iochain.h"
+
+
+#ifndef BSHUF_MIN_RECOMMEND_BLOCK
+#define BSHUF_MIN_RECOMMEND_BLOCK 128
+#define BSHUF_BLOCKED_MULT 8
+#define BSHUF_TARGET_BLOCK_SIZE_B 8192
+#endif
+
+
+
+#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; }
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ---- Utility functions for internal use only ---- */
+
+int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
+        const size_t elem_size);
+
+/* Read a 32 bit unsigned integer from a buffer big endian order. */
+uint32_t bshuf_read_uint32_BE(const void* buf);
+
+/* Write a 32 bit unsigned integer to a buffer in big endian order. */
+void bshuf_write_uint32_BE(void* buf, uint32_t num);
+
+int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
+        const size_t elem_size);
+
+/* Function definition for worker functions that process a single block. */
+typedef int64_t (*bshufBlockFunDef)(ioc_chain* C_ptr,
+        const size_t size, const size_t elem_size);
+
+/* Wrap a function for processing a single block to process an entire buffer in
+ * parallel. */
+int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,
+        const size_t size, const size_t elem_size, size_t block_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

bslz4/src/iochain.c

@@ -0,0 +1,90 @@
+/*
+ * IOchain - Distribute a chain of dependant IO events amoung threads.
+ *
+ * This file is part of Bitshuffle
+ * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
+ * Website: http://www.github.com/kiyo-masui/bitshuffle
+ * Created: 2014
+ *
+ * See LICENSE file for details about copyright and rights to use.
+ *
+ */
+
+#include <stdlib.h>
+#include "iochain.h"
+
+
+void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0) {
+#ifdef _OPENMP
+    omp_init_lock(&C->next_lock);
+    for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
+        omp_init_lock(&(C->in_pl[ii].lock));
+        omp_init_lock(&(C->out_pl[ii].lock));
+    }
+#endif
+    C->next = 0;
+    C->in_pl[0].ptr = in_ptr_0;
+    C->out_pl[0].ptr = out_ptr_0;
+}
+
+
+void ioc_destroy(ioc_chain *C) {
+#ifdef _OPENMP
+    omp_destroy_lock(&C->next_lock);
+    for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
+        omp_destroy_lock(&(C->in_pl[ii].lock));
+        omp_destroy_lock(&(C->out_pl[ii].lock));
+    }
+#endif
+}
+
+
+const void * ioc_get_in(ioc_chain *C, size_t *this_iter) {
+#ifdef _OPENMP
+    omp_set_lock(&C->next_lock);
+    #pragma omp flush
+#endif
+    *this_iter = C->next;
+    C->next ++;
+#ifdef _OPENMP
+    omp_set_lock(&(C->in_pl[*this_iter % IOC_SIZE].lock));
+    omp_set_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
+    omp_set_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
+    omp_unset_lock(&C->next_lock);
+#endif
+    return C->in_pl[*this_iter % IOC_SIZE].ptr;
+}
+
+
+void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr) {
+    C->in_pl[(*this_iter + 1) % IOC_SIZE].ptr = in_ptr;
+#ifdef _OPENMP
+    omp_unset_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
+#endif
+}
+
+
+void * ioc_get_out(ioc_chain *C, size_t *this_iter) {
+#ifdef _OPENMP
+    omp_set_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
+    #pragma omp flush
+#endif
+    void *out_ptr = C->out_pl[*this_iter % IOC_SIZE].ptr;
+#ifdef _OPENMP
+    omp_unset_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
+#endif
+    return out_ptr;
+}
+
+
+void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) {
+    C->out_pl[(*this_iter + 1) % IOC_SIZE].ptr = out_ptr;
+#ifdef _OPENMP
+    omp_unset_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
+
+
+
+    omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock));
+#endif
+}
+

bslz4/src/iochain.h

@@ -0,0 +1,94 @@
+/*
+ * IOchain - Distribute a chain of dependant IO events amoung threads.
+ *
+ * This file is part of Bitshuffle
+ * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
+ * Website: http://www.github.com/kiyo-masui/bitshuffle
+ * Created: 2014
+ *
+ * See LICENSE file for details about copyright and rights to use.
+ *
+ *
+ * Header File
+ *
+ * Similar in concept to a queue. Each task includes reading an input
+ * and writing output, but the location of the input/output (the pointers)
+ * depend on the previous item in the chain.
+ *
+ * This is designed for parallelizing blocked compression/decompression IO,
+ * where the destination of a compressed block depends on the compressed size
+ * of all previous blocks.
+ *
+ * Implemented with OpenMP locks.
+ *
+ *
+ * Usage
+ * -----
+ *  - Call `ioc_init` in serial block.
+ *  - Each thread should create a local variable *size_t this_iter* and
+ *    pass its address to all function calls. Its value will be set
+ *    inside the functions and is used to identify the thread.
+ *  - Each thread must call each of the `ioc_get*` and `ioc_set*` methods
+ *    exactly once per iteration, starting with `ioc_get_in` and ending
+ *    with `ioc_set_next_out`.
+ *  - The order (`ioc_get_in`, `ioc_set_next_in`, *work*, `ioc_get_out`,
+ *    `ioc_set_next_out`, *work*) is most efficient.
+ *  - Have each thread call `ioc_end_pop`.
+ *  - `ioc_get_in` is blocked until the previous entry's
+ *    `ioc_set_next_in` is called.
+ *  - `ioc_get_out` is blocked until the previous entry's
+ *    `ioc_set_next_out` is called.
+ *  - There are no blocks on the very first iteration.
+ *  - Call `ioc_destroy` in serial block.
+ *  - Safe for num_threads >= IOC_SIZE (but less efficient).
+ *
+ */
+
+
+#ifndef IOCHAIN_H
+#define IOCHAIN_H
+
+
+#include <stdlib.h>
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+
+
+#define IOC_SIZE 33
+
+
+typedef struct ioc_ptr_and_lock {
+#ifdef _OPENMP
+    omp_lock_t lock;
+#endif
+    void *ptr;
+} ptr_and_lock;
+
+typedef struct ioc_const_ptr_and_lock {
+#ifdef _OPENMP
+    omp_lock_t lock;
+#endif
+    const void *ptr;
+} const_ptr_and_lock;
+
+
+typedef struct ioc_chain {
+#ifdef _OPENMP
+    omp_lock_t next_lock;
+#endif
+    size_t next;
+    const_ptr_and_lock in_pl[IOC_SIZE];
+    ptr_and_lock out_pl[IOC_SIZE];
+} ioc_chain;
+
+
+void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0);
+void ioc_destroy(ioc_chain *C);
+const void * ioc_get_in(ioc_chain *C, size_t *this_iter);
+void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr);
+void * ioc_get_out(ioc_chain *C, size_t *this_iter);
+void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr);
+
+#endif
+

bslz4/src/lz4.h

@@ -0,0 +1,360 @@
+/*
+   LZ4 - Fast LZ compression algorithm
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * lz4.h provides block compression functions, and gives full buffer control to programmer.
+ * If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
+ * and can let the library handle its own memory, please use lz4frame.h instead.
+*/
+
+/**************************************
+*  Version
+**************************************/
+#define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
+#define LZ4_VERSION_MINOR    7    /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
+int LZ4_versionNumber (void);
+
+/**************************************
+*  Tuning parameter
+**************************************/
+/*
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+ */
+#define LZ4_MEMORY_USAGE 14
+
+
+/**************************************
+*  Simple Functions
+**************************************/
+
+int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
+int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+
+/*
+LZ4_compress_default() :
+    Compresses 'sourceSize' bytes from buffer 'source'
+    into already allocated 'dest' buffer of size 'maxDestSize'.
+    Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
+    It also runs faster, so it's a recommended setting.
+    If the function cannot compress 'source' into a more limited 'dest' budget,
+    compression stops *immediately*, and the function result is zero.
+    As a consequence, 'dest' content is not valid.
+    This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
+        sourceSize  : Max supported value is LZ4_MAX_INPUT_VALUE
+        maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
+        return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
+              or 0 if compression fails
+
+LZ4_decompress_safe() :
+    compressedSize : is the precise full size of the compressed block.
+    maxDecompressedSize : is the size of destination buffer, which must be already allocated.
+    return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
+             If destination buffer is not large enough, decoding will stop and output an error code (<0).
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             This function is protected against buffer overflow exploits, including malicious data packets.
+             It never writes outside output buffer, nor reads outside input buffer.
+*/
+
+
+/**************************************
+*  Advanced Functions
+**************************************/
+#define LZ4_MAX_INPUT_SIZE        0x7E000000   /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize)  ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
+
+/*
+LZ4_compressBound() :
+    Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
+    This function is primarily useful for memory allocation purposes (destination buffer size).
+    Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
+    Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+        inputSize  : max supported value is LZ4_MAX_INPUT_SIZE
+        return : maximum output size in a "worst case" scenario
+              or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+*/
+int LZ4_compressBound(int inputSize);
+
+/*
+LZ4_compress_fast() :
+    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+    The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
+    It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
+    An acceleration value of "1" is the same as regular LZ4_compress_default()
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+*/
+int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+
+
+/*
+LZ4_compress_fast_extState() :
+    Same compression function, just using an externally allocated memory space to store compression state.
+    Use LZ4_sizeofState() to know how much memory must be allocated,
+    and allocate it on 8-bytes boundaries (using malloc() typically).
+    Then, provide it as 'void* state' to compression function.
+*/
+int LZ4_sizeofState(void);
+int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
+
+
+/*
+LZ4_compress_destSize() :
+    Reverse the logic, by compressing as much data as possible from 'source' buffer
+    into already allocated buffer 'dest' of size 'targetDestSize'.
+    This function either compresses the entire 'source' content into 'dest' if it's large enough,
+    or fill 'dest' buffer completely with as much data as possible from 'source'.
+        *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
+                         New value is necessarily <= old value.
+        return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
+              or 0 if compression fails
+*/
+int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
+
+
+/*
+LZ4_decompress_fast() :
+    originalSize : is the original and therefore uncompressed size
+    return : the number of bytes read from the source buffer (in other words, the compressed size)
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
+    note : This function fully respect memory boundaries for properly formed compressed data.
+           It is a bit faster than LZ4_decompress_safe().
+           However, it does not provide any protection against intentionally modified data stream (malicious input).
+           Use this function in trusted environment only (data to decode comes from a trusted source).
+*/
+int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
+
+/*
+LZ4_decompress_safe_partial() :
+    This function decompress a compressed block of size 'compressedSize' at position 'source'
+    into destination buffer 'dest' of size 'maxDecompressedSize'.
+    The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+    reducing decompression time.
+    return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
+       Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+             Always control how many bytes were decoded.
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
+
+
+/***********************************************
+*  Streaming Compression Functions
+***********************************************/
+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE     (LZ4_STREAMSIZE_U64 * sizeof(long long))
+/*
+ * LZ4_stream_t
+ * information structure to track an LZ4 stream.
+ * important : init this structure content before first use !
+ * note : only allocated directly the structure if you are statically linking LZ4
+ *        If you are using liblz4 as a DLL, please use below construction methods instead.
+ */
+typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
+
+/*
+ * LZ4_resetStream
+ * Use this function to init an allocated LZ4_stream_t structure
+ */
+void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
+/*
+ * LZ4_createStream will allocate and initialize an LZ4_stream_t structure
+ * LZ4_freeStream releases its memory.
+ * In the context of a DLL (liblz4), please use these methods rather than the static struct.
+ * They are more future proof, in case of a change of LZ4_stream_t size.
+ */
+LZ4_stream_t* LZ4_createStream(void);
+int           LZ4_freeStream (LZ4_stream_t* streamPtr);
+
+/*
+ * LZ4_loadDict
+ * Use this function to load a static dictionary into LZ4_stream.
+ * Any previous data will be forgotten, only 'dictionary' will remain in memory.
+ * Loading a size of 0 is allowed.
+ * Return : dictionary size, in bytes (necessarily <= 64 KB)
+ */
+int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
+
+/*
+ * LZ4_compress_fast_continue
+ * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
+ * Important : Previous data blocks are assumed to still be present and unmodified !
+ * 'dst' buffer must be already allocated.
+ * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ */
+int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+
+/*
+ * LZ4_saveDict
+ * If previously compressed data block is not guaranteed to remain available at its memory location
+ * save it into a safer place (char* safeBuffer)
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ *        dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
+ */
+int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
+
+
+/************************************************
+*  Streaming Decompression Functions
+************************************************/
+
+#define LZ4_STREAMDECODESIZE_U64  4
+#define LZ4_STREAMDECODESIZE     (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t;
+/*
+ * LZ4_streamDecode_t
+ * information structure to track an LZ4 stream.
+ * init this structure content using LZ4_setStreamDecode or memset() before first use !
+ *
+ * In the context of a DLL (liblz4) please prefer usage of construction methods below.
+ * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
+ * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
+ * LZ4_freeStreamDecode releases its memory.
+ */
+LZ4_streamDecode_t* LZ4_createStreamDecode(void);
+int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+
+/*
+ * LZ4_setStreamDecode
+ * Use this function to instruct where to find the dictionary.
+ * Setting a size of 0 is allowed (same effect as reset).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
+
+/*
+*_continue() :
+    These decoding functions allow decompression of multiple blocks in "streaming" mode.
+    Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+    In the case of a ring buffers, decoding buffer must be either :
+    - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+      In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+    - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+      maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+      In which case, encoding and decoding buffers do not need to be synchronized,
+      and encoding ring buffer can have any size, including small ones ( < 64 KB).
+    - _At least_ 64 KB + 8 bytes + maxBlockSize.
+      In which case, encoding and decoding buffers do not need to be synchronized,
+      and encoding ring buffer can have any size, including larger than decoding buffer.
+    Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+    and indicate where it is saved using LZ4_setStreamDecode()
+*/
+int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
+
+
+/*
+Advanced decoding functions :
+*_usingDict() :
+    These decoding functions work the same as
+    a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
+    They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
+*/
+int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
+int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+
+
+
+/**************************************
+*  Obsolete Functions
+**************************************/
+/* Deprecate Warnings */
+/* Should these warnings messages be a problem,
+   it is generally possible to disable them,
+   with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual for example.
+   You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
+#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ4_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#  if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (LZ4_GCC_VERSION >= 301)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define LZ4_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
+#    define LZ4_DEPRECATED(message)
+#  endif
+#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */
+
+/* Obsolete compression functions */
+/* These functions are planned to start generate warnings by r131 approximately */
+int LZ4_compress               (const char* source, char* dest, int sourceSize);
+int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Obsolete decompression functions */
+/* These function names are completely deprecated and must no longer be used.
+   They are only provided here for compatibility with older programs.
+    - LZ4_uncompress is the same as LZ4_decompress_fast
+    - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
+   These function prototypes are now disabled; uncomment them only if you really need them.
+   It is highly recommended to stop using these prototypes and migrate to maintained ones */
+/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
+/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStream() instead") int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("use LZ4_resetStream() instead")  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInputBuffer (void* state);
+
+/* Obsolete streaming decoding functions */
+LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+
+
+#if defined (__cplusplus)
+}
+#endif

src/file.c

@@ -5,29 +5,15 @@
 
 
 #include <hdf5.h>
+#include <hdf5_hl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
 #include "file.h"
 #include "err.h"
+#include "filters.h"
 
-hid_t h5_int_type_from_width(const int width) {
-	if (width == sizeof(char)) {
-		return H5T_NATIVE_SCHAR;
-	} else if (width == sizeof(short)) {
-		return H5T_NATIVE_SHORT;
-	} else if (width == sizeof(int)) {
-		return H5T_NATIVE_INT;
-	} else if (width == sizeof(long)) {
-		return H5T_NATIVE_LONG;
-	} else if (width == sizeof(long long)) {
-		return H5T_NATIVE_LLONG;
-	} else {
-		/* TODO: error */
-		return -1;
-	}
-}
 
 void clear_det_visit_objects(struct det_visit_objects_t *objects) {
 	if (objects->nxdata) {
@@ -41,18 +27,24 @@ void clear_det_visit_objects(struct det_visit_objects_t *objects) {
 }
 
 
-void free_nxs_data_description(struct data_description_t *desc) {
+void free_ds_desc(struct ds_desc_t *desc) {
-	if (desc->extra) free(desc->extra); /* should just be NULL */
+	H5Gclose(desc->det_g_id);
-	desc->extra = NULL;
+	H5Gclose(desc->data_g_id);
+	free(desc);
 }
 
+void free_nxs_desc(struct ds_desc_t *desc) {
+	free_ds_desc(desc);
+}
 
-void free_eiger_data_description(struct data_description_t *desc) {
+void free_eiger_desc(struct ds_desc_t *desc) {
-	if (!desc->extra) return;
+	struct eiger_ds_desc_t *e_desc = (struct eiger_ds_desc_t *) desc;
-	struct eiger_data_description_t *extra = desc->extra;
+	free(e_desc->block_sizes);
-	if (extra->block_sizes) free(extra->block_sizes);
+	free_ds_desc(desc);
-	free(extra);
+}
-	desc->extra = NULL;
+
+void free_opt_eiger_desc(struct ds_desc_t *desc) {
+	free_eiger_desc(desc);
 }
 
 
@@ -80,13 +72,12 @@ double scale_from_units(const char* unit_string) {
 	}
 }
 
-int get_nxs_dataset_dims(const struct data_description_t *desc, struct dataset_properties_t *properties) {
+int get_nxs_dataset_dims(struct ds_desc_t *desc) {
 	hid_t g_id, ds_id, s_id, t_id;
 	int retval = 0;
 	int ndims = 0;
 	int width = 0;
-	hsize_t dims[3] = {0};
+	g_id = desc->data_g_id;;
-	g_id = desc->data_group_id;;
 
 	ds_id = H5Dopen2(g_id, "data", H5P_DEFAULT);
 	if (ds_id <= 0) {
@@ -115,12 +106,11 @@ int get_nxs_dataset_dims(const struct data_description_t *desc, struct dataset_p
 		ERROR_JUMP(-1, close_space, message);
 	}
 
-	if (H5Sget_simple_extent_dims(s_id, dims, NULL) < 0) {
+	if (H5Sget_simple_extent_dims(s_id, desc->dims, NULL) < 0) {
 		ERROR_JUMP(-1, close_space, "Error getting dataset dimensions");
 	}
 
-	memcpy(properties->dims, dims, 3 * sizeof(*dims));
+	desc->data_width = width;
-	properties->data_width = width;
 
 close_space:
 	H5Sclose(s_id);
@@ -132,10 +122,18 @@ done:
 	return retval;
 }
 
-int get_frame(hid_t g_id, const char* name, hsize_t *frame_idx, hsize_t *frame_size, int data_width, void *buffer) {
+int get_frame_simple(const struct ds_desc_t *desc,
+		const char *name,
+		const hsize_t *frame_idx,
+		const hsize_t *frame_size,
+		void *buffer) {
+
 	int retval = 0;
 	herr_t err = 0;
-	hid_t ds_id, s_id, ms_id, t_id;
+	hid_t g_id, ds_id, s_id, ms_id, t_id;
+
+	g_id = desc->data_g_id;
+
 	ds_id = H5Dopen2(g_id, name, H5P_DEFAULT);
 	if (ds_id <= 0) {
 		char message[64];
@@ -146,6 +144,10 @@ int get_frame(hid_t g_id, const char* name, hsize_t *frame_idx, hsize_t *frame_s
 	if (s_id <= 0) {
 		ERROR_JUMP(-1, close_dataset, "Error getting dataspace");
 	}
+	t_id = H5Dget_type(ds_id);
+	if (t_id <= 0) {
+		ERROR_JUMP(-1, close_type, "Error retrieving datatype");
+	}
 	err = H5Sselect_hyperslab(s_id, H5S_SELECT_SET, frame_idx, NULL, frame_size, NULL);
 	if (err < 0) {
 		ERROR_JUMP(-1, close_space, "Error seleting hyperslab");
@@ -155,12 +157,6 @@ int get_frame(hid_t g_id, const char* name, hsize_t *frame_idx, hsize_t *frame_s
 		ERROR_JUMP(-1, close_space, "Could not create dataspace");
 	}
 
-	t_id = h5_int_type_from_width(data_width);
-	if (t_id < 0) {
-		char message[64];
-		sprintf(message, "Could not infer signed integer from width %d", data_width);
-		ERROR_JUMP(-1, close_mspace, message);
-	}
 	err = H5Dread(ds_id, t_id, ms_id, s_id, H5P_DEFAULT, buffer);
 	if (err < 0) {
 		ERROR_JUMP(-1, close_mspace, "Error reading dataset");
@@ -170,29 +166,110 @@ close_mspace:
 	H5Sclose(ms_id);
 close_space:
 	H5Sclose(s_id);
+close_type:
+	H5Tclose(t_id);
 close_dataset:
 	H5Dclose(ds_id);
 done:
 	return retval;
 }
 
+
+int get_frame_from_chunk(const struct ds_desc_t *desc,
+		const char *ds_name,
+		const hsize_t *frame_idx,
+		const hsize_t *frame_size,
+		void *buffer) {
+
+	hid_t d_id = 0;
+	hsize_t c_offset[3] = {frame_idx[0], 0, 0};
+	uint32_t c_filter_mask = 0;
+	hsize_t c_bytes;
+	void *c_buffer = NULL;
+	const struct opt_eiger_ds_desc_t *o_eiger_desc = (struct opt_eiger_ds_desc_t *) desc;
+	int retval = 0;
+
+	if (frame_idx[1] != 0 || frame_idx[2] != 0) {
+		char message[64];
+		sprintf(message, "Require frame selection starts at [n, 0, 0], not [n, %llu, %llu]",
+				frame_idx[1], frame_idx[2]);
+		ERROR_JUMP(-1, done, message);
+	}
+
+	d_id = H5Dopen(desc->data_g_id, ds_name, H5P_DEFAULT);
+	if (d_id < 0) {
+		char message[64];
+		sprintf(message, "Error opening dataset %.32s", ds_name);
+		ERROR_JUMP(-1, done, message);
+	}
+
+
+	if (H5Dget_chunk_storage_size(d_id, c_offset, &c_bytes) < 0) {
+		char message[96];
+		sprintf(message, "Error reading chunk size from %.32s for frame %llu", ds_name, frame_idx[0]);
+		ERROR_JUMP(-1, done, message);
+	}
+	if (c_bytes == 0) {
+		char message[96];
+		sprintf(message, "Target chunk %llu has zero size for dataset %.32s", frame_idx[0], ds_name);
+		ERROR_JUMP(-1, done, message);
+	}
+
+	if (o_eiger_desc->bs_applied) {
+		c_buffer = malloc(c_bytes);
+		if (!c_buffer) {
+			char message[128];
+			sprintf(message, "Unable to allocate chunk buffer for dataset %.32s - frame %llu, size %llu bytes",
+					ds_name, frame_idx[0], c_bytes);
+			ERROR_JUMP(-1, done, message);
+		}
+	} else {
+		c_buffer = buffer;
+	}
+
+	if (H5DOread_chunk(d_id, H5P_DEFAULT, c_offset, &c_filter_mask, c_buffer) < 0) {
+		char message[128];
+		sprintf(message, "Error reading chunk %llu from dataset %.32s - size %llu bytes",
+				frame_idx[0], ds_name, c_bytes);
+		ERROR_JUMP(-1, done, message);
+	}
+
+	if (o_eiger_desc->bs_applied) {
+		if (bslz4_decompress(
+					o_eiger_desc->bs_params,
+					c_bytes,
+					c_buffer,
+					desc->data_width * frame_size[1] * frame_size[2],
+					buffer) < 0) {
+			char message[128];
+			sprintf(message, "Error processing chunk %llu from %.32s with bitshuffle_lz4",
+					frame_idx[0], ds_name);
+			ERROR_JUMP(-1, done, message);
+		}
+	}
+
+done:
+	if (c_buffer && (c_buffer != buffer)) free(c_buffer);
+	if (d_id) H5Dclose(d_id);
+	return retval;
+}
+
+
 int get_nxs_frame(
-		const struct data_description_t *desc,
+		const struct ds_desc_t *desc,
-		const struct dataset_properties_t *ds_prop,
+		const int n,
-		int n,
-		int data_width,
 		void *buffer) {
 	/* detector data are the two inner most indices */
 	/* TODO: handle ndims > 3 and select appropriately */
 	int retval = 0;
 	hsize_t frame_idx[3] = {n, 0, 0};
-	hsize_t frame_size[3] = {1, ds_prop->dims[1], ds_prop->dims[2]};
+	hsize_t frame_size[3] = {1, desc->dims[1], desc->dims[2]};
-	if (n < 0 || n >= ds_prop->dims[0]) {
+	if (n < 0 || n >= desc->dims[0]) {
 		char message[64];
-		sprintf(message, "Selected frame %d is out of range valid range [0, %d]", n, (int) ds_prop->dims[0] - 1);
+		sprintf(message, "Selected frame %d is out of range valid range [0, %d]", n, (int) desc->dims[0] - 1);
 		ERROR_JUMP(-1, done, message);
 	}
-	retval = get_frame(desc->data_group_id, "data", frame_idx, frame_size, data_width, buffer);
+	retval = get_frame_simple(desc, "data", frame_idx, frame_size, buffer);
 	if (retval < 0) {
 		ERROR_JUMP(retval, done, "");
 	}
@@ -202,22 +279,20 @@ done:
 
 
 int get_dectris_eiger_frame(
-		const struct data_description_t *desc,
+		const struct ds_desc_t *desc,
-		const struct dataset_properties_t *ds_prop,
 		int n,
-		int data_width,
 		void *buffer) {
 
 	int retval = 0;
 	int block, frame_count, idx;
-	struct eiger_data_description_t *eiger_desc = desc->extra;
+	struct eiger_ds_desc_t *eiger_desc = (struct eiger_ds_desc_t*) desc;
 	char data_name[16] = {0};
 	hsize_t frame_idx[3] = {0, 0, 0};
-	hsize_t frame_size[3] = {1, ds_prop->dims[1], ds_prop->dims[2]};
+	hsize_t frame_size[3] = {1, desc->dims[1], desc->dims[2]};
 
-	if (n < 0 || n >= ds_prop->dims[0]) {
+	if (n < 0 || n >= desc->dims[0]) {
 		char message[64];
-		sprintf(message, "Selected frame %d is out of range valid range [0, %d]", n, (int) ds_prop->dims[0] - 1);
+		sprintf(message, "Selected frame %d is out of range valid range [0, %d]", n, (int) desc->dims[0] - 1);
 		ERROR_JUMP(-1, done, message);
 	}
 
@@ -228,7 +303,7 @@ int get_dectris_eiger_frame(
 	idx = n - (frame_count - eiger_desc->block_sizes[block]); /* index in current block */
 	frame_idx[0] = idx;
 	sprintf(data_name, "data_%06d", block + 1);
-	retval = get_frame(desc->data_group_id, data_name, frame_idx, frame_size, data_width, buffer);
+	retval = eiger_desc->frame_func(desc, data_name, frame_idx, frame_size, buffer);
 	if (retval < 0) {
 		ERROR_JUMP(retval, done, "");
 	}
@@ -237,7 +312,7 @@ done:
 }
 
 
-int get_dectris_eiger_dataset_dims(const struct data_description_t *desc, struct dataset_properties_t *properties) {
+int get_dectris_eiger_dataset_dims(struct ds_desc_t *desc) {
 	int retval = 0;
 	int n_datas = 0;
 	int n = 0;
@@ -246,11 +321,12 @@ int get_dectris_eiger_dataset_dims(const struct data_description_t *desc, struct
 	char ds_name[16] = {0}; /* 12 chars in "data_xxxxxx\0" */
 	int *frame_counts = NULL;
 	hsize_t dims[3] = {0};
+	struct eiger_ds_desc_t *eiger_desc = (struct eiger_ds_desc_t*) desc;
 
 	/* datasets are "data_%06d % n" - need to determine how many of these there are and what the ranges are */
 
 	sprintf(ds_name, "data_%06d", n_datas + 1);
-	while (H5Lexists(desc->data_group_id, ds_name, H5P_DEFAULT) > 0) {
+	while (H5Lexists(desc->data_g_id, ds_name, H5P_DEFAULT) > 0) {
 		sprintf(ds_name, "data_%06d", ++n_datas + 1);
 	}
 
@@ -260,7 +336,7 @@ int get_dectris_eiger_dataset_dims(const struct data_description_t *desc, struct
 		hid_t ds_id, t_id, s_id;
 		hsize_t block_dims[3] = {0};
 		sprintf(ds_name, "data_%06d", n + 1);
-		ds_id = H5Dopen2(desc->data_group_id, ds_name, H5P_DEFAULT);
+		ds_id = H5Dopen2(desc->data_g_id, ds_name, H5P_DEFAULT);
 		if (ds_id < 0) {
 			char message[64];
 			sprintf("Unable to open dataset %.16s", ds_name);
@@ -309,10 +385,10 @@ loop_end:
 	if (retval < 0) {
 		free(frame_counts);
 	} else {
-		memcpy(properties->dims, dims, 3 * sizeof(*dims));
+		memcpy(desc->dims, dims, 3 * sizeof(*dims));
-		properties->data_width = data_width;
+		desc->data_width = data_width;
-		((struct eiger_data_description_t *) desc->extra)->n_data_blocks = n_datas;
+		eiger_desc->n_data_blocks = n_datas;
-		((struct eiger_data_description_t *) desc->extra)->block_sizes = frame_counts;
+		eiger_desc->block_sizes = frame_counts;
 	}
 	return retval;
 }
@@ -424,12 +500,12 @@ done:
 }
 
 
-int get_nxs_pixel_info(const struct data_description_t *desc, double *x_size, double *y_size) {
+int get_nxs_pixel_info(const struct ds_desc_t *desc, double *x_size, double *y_size) {
 	int retval = 0;
-	if (read_pixel_info(desc->det_group_id, "x_pixel_size", x_size) < 0) {
+	if (read_pixel_info(desc->det_g_id, "x_pixel_size", x_size) < 0) {
 		ERROR_JUMP(-1, done, "");
 	}
-	if (read_pixel_info(desc->det_group_id, "y_pixel_size", y_size) < 0) {
+	if (read_pixel_info(desc->det_g_id, "y_pixel_size", y_size) < 0) {
 		ERROR_JUMP(-1, done, "");
 	}
 done:
@@ -437,12 +513,12 @@ done:
 }
 
 
-int get_dectris_eiger_pixel_info(const struct data_description_t *desc, double *x_size, double *y_size) {
+int get_dectris_eiger_pixel_info(const struct ds_desc_t *desc, double *x_size, double *y_size) {
 	int retval = 0;
-	if (read_pixel_info(desc->det_group_id, "detectorSpecific/x_pixel_size", x_size) < 0) {
+	if (read_pixel_info(desc->det_g_id, "detectorSpecific/x_pixel_size", x_size) < 0) {
 		ERROR_JUMP(-1, done, "");
 	}
-	if (read_pixel_info(desc->det_group_id, "detectorSpecific/y_pixel_size", y_size) < 0) {
+	if (read_pixel_info(desc->det_g_id, "detectorSpecific/y_pixel_size", y_size) < 0) {
 		ERROR_JUMP(-1, done, "");
 	}
 done:
@@ -450,12 +526,12 @@ done:
 }
 
 
-int get_nxs_pixel_mask(const struct data_description_t *desc, int *buffer) {
+int get_nxs_pixel_mask(const struct ds_desc_t *desc, int *buffer) {
 	int retval = 0;
 	hid_t ds_id;
 	herr_t err = 0;
 
-	ds_id = H5Dopen2(desc->det_group_id, "pixel_mask", H5P_DEFAULT);
+	ds_id = H5Dopen2(desc->det_g_id, "pixel_mask", H5P_DEFAULT);
 	if (ds_id < 0) {
 		ERROR_JUMP(-1, done, "Error opening pixel_mask dataset");
 	}
@@ -472,12 +548,12 @@ done:
 }
 
 
-int get_dectris_eiger_pixel_mask(const struct data_description_t *desc, int *buffer) {
+int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) {
 	int retval = 0;
 	hid_t ds_id;
 	herr_t err = 0;
 
-	ds_id = H5Dopen2(desc->det_group_id, "detectorSpecific/pixel_mask", H5P_DEFAULT);
+	ds_id = H5Dopen2(desc->det_g_id, "detectorSpecific/pixel_mask", H5P_DEFAULT);
 	if (ds_id < 0) {
 		ERROR_JUMP(-1, done, "Error opening detectorSpecific/pixel_mask");
 	}
@@ -491,7 +567,13 @@ close_dataset:
 	H5Dclose(ds_id);
 done:
 	return retval;
+}
 
+
+int get_null_pixel_mask(const struct ds_desc_t *desc, int *buffer) {
+	hsize_t buffer_length = desc->dims[1] * desc->dims[2];
+	memset(buffer, 0, sizeof(*buffer) * buffer_length);
+	return 0;
 }
 
 
@@ -602,85 +684,228 @@ done:
 	return retval;
 }
 
+int check_for_chunk_read(
+		hid_t g_id,
+		const char* ds_name,
+		struct opt_eiger_ds_desc_t *desc) {
 
-int fill_data_descriptor(struct data_description_t *data_desc, struct det_visit_objects_t *visit_result) {
 	int retval = 0;
-	data_desc->det_group_id = visit_result->nxdetector;
+	int n_filters;
+	hsize_t cdims[3];
+	hid_t ds_id, dcpl, s_id;
+	unsigned int filter_flags, filter_config;
+	char filter_name[16];
+	size_t name_len = 16;
+	size_t cd_nelems = BS_H5_N_PARAMS;
+	hsize_t dims[3];
+	H5Z_filter_t filter;
+
+	dcpl = 0;
+	s_id = 0;
+	ds_id = 0;
+
+	ds_id = H5Dopen2(g_id, ds_name, H5P_DEFAULT);
+	if (ds_id < 0) {
+		char message[64];
+		sprintf(message, "Error opening dataset %.32s", ds_name);
+		ERROR_JUMP(-1, done, message)
+	}
+
+	s_id = H5Dget_space(ds_id);
+	if (s_id < 0) {
+		char message[64];
+		sprintf(message, "Error opening dataspace for %.32s", ds_name);
+		ERROR_JUMP(-1, done, message);
+	}
+
+	if (3 != H5Sget_simple_extent_ndims(s_id)) {
+		goto done;
+	}
+
+	if (H5Sget_simple_extent_dims(s_id, dims, NULL) < 0) {
+		char message[80];
+		sprintf(message, "Error retriving dataset dimensions for %.32s", ds_name);
+		ERROR_JUMP(-1, done, message);
+	}
+
+	dcpl = H5Dget_create_plist(ds_id);
+	if (dcpl < 0) {
+		ERROR_JUMP(-1, done, "Error getting dataset creation property list");
+	}
+
+	/* check the chunk layout matches the layout we expect of
+	 * [1, frame_size_y, frame_size_x] (1 frame == 1 chunk) */
+	int cndims = H5Pget_chunk(dcpl, 3, cdims);
+	if (cndims != 3) {
+		goto done;
+	}
+	if (cdims[0] != 1 || cdims[1] != dims[1] || cdims[2] != dims[2]) {
+		goto done;
+	}
+
+	/* check for potential filters - only the bitshuffle filter is supported */
+	n_filters = H5Pget_nfilters(dcpl);
+	if (n_filters < 0) {
+		ERROR_JUMP(-1, done, "Error retrieving number of filters on dataset");
+	} else if (n_filters > 1) {
+		goto done;
+	}
+
+	if (n_filters == 1) {
+		filter = H5Pget_filter2(dcpl, 0, &filter_flags,
+				&cd_nelems, desc->bs_params,
+				name_len, filter_name,
+				&filter_config);
+		if (filter < 0) {
+			ERROR_JUMP(-1, done, "Error retrieving filter information");
+		}
+		if (filter != BS_H5_FILTER_ID) {
+			goto done;
+		}
+		if (cd_nelems > BS_H5_N_PARAMS) {
+			char message[128];
+			sprintf(message,
+					"More than expected number of parameters to bitshuffle filter - expected %d, was %lu",
+					BS_H5_N_PARAMS, cd_nelems);
+			ERROR_JUMP(-1, done, message);
+		}
+		desc->bs_applied = 1;
+	} else {
+		desc->bs_applied = 0;
+	}
+
+	retval = 1;
+
+done:
+	if (dcpl) H5Pclose(dcpl);
+	if (s_id) H5Sclose(s_id);
+	if (ds_id) H5Dclose(ds_id);
+	return retval;
+}
+
+int create_dataset_descriptor(struct ds_desc_t **desc, struct det_visit_objects_t *visit_result) {
+	int retval = 0;
+	hid_t g_id, ds_id;
+	int (*pxl_func)(const struct ds_desc_t*, double*, double*);
+	int (*pxl_mask_func)(const struct ds_desc_t*, int*);
+	int (*ds_prop_func)(struct ds_desc_t*);
+	int (*frame_func)(const struct ds_desc_t*, int, void*);
+	void (*free_func)(struct ds_desc_t*);
+	struct ds_desc_t *output;
+
+	g_id = visit_result->nxdetector;
 
 	/* determine the pixel information location */
-	if (H5Lexists(data_desc->det_group_id, "x_pixel_size", H5P_DEFAULT) > 0 &&
+	if (H5Lexists(g_id, "x_pixel_size", H5P_DEFAULT) > 0 &&
-			H5Lexists(data_desc->det_group_id, "y_pixel_size", H5P_DEFAULT)) {
+			H5Lexists(g_id, "y_pixel_size", H5P_DEFAULT)) {
-		data_desc->get_pixel_properties = &get_nxs_pixel_info;
+		pxl_func = &get_nxs_pixel_info;
-	} else if (H5Lexists(data_desc->det_group_id, "detectorSpecific", H5P_DEFAULT) > 0 &&
+	} else if (H5Lexists(g_id, "detectorSpecific", H5P_DEFAULT) > 0 &&
-			H5Lexists(data_desc->det_group_id, "detectorSpecific/x_pixel_size", H5P_DEFAULT) > 0 &&
+			H5Lexists(g_id, "detectorSpecific/x_pixel_size", H5P_DEFAULT) > 0 &&
-			H5Lexists(data_desc->det_group_id, "detectorSpecific/y_pixel_size", H5P_DEFAULT) > 0) {
+			H5Lexists(g_id, "detectorSpecific/y_pixel_size", H5P_DEFAULT) > 0) {
-		data_desc->get_pixel_properties = &get_dectris_eiger_pixel_info;
+		pxl_func = &get_dectris_eiger_pixel_info;
 	} else {
-		data_desc->get_pixel_properties = NULL;
 		ERROR_JUMP(-1, done, "Could not locate x_pixel_size and y_pixel_size");
 	}
 
 	/* determine pixel mask location */
-	if (H5Lexists(data_desc->det_group_id, "pixel_mask", H5P_DEFAULT) > 0) {
+	if (H5Lexists(g_id, "pixel_mask", H5P_DEFAULT) > 0) {
-		data_desc->get_pixel_mask = &get_nxs_pixel_mask;
+		pxl_mask_func = &get_nxs_pixel_mask;
-	} else if (H5Lexists(data_desc->det_group_id, "detectorSpecific", H5P_DEFAULT) > 0 &&
+	} else if (H5Lexists(g_id, "detectorSpecific", H5P_DEFAULT) > 0 &&
-			H5Lexists(data_desc->det_group_id, "detectorSpecific/pixel_mask", H5P_DEFAULT) > 0) {
+			H5Lexists(g_id, "detectorSpecific/pixel_mask", H5P_DEFAULT) > 0) {
-		data_desc->get_pixel_mask = &get_dectris_eiger_pixel_mask;
+		pxl_mask_func = &get_dectris_eiger_pixel_mask;
 	} else {
-		data_desc->get_pixel_mask = NULL;
+		pxl_mask_func = &get_null_pixel_mask;
-		ERROR_JUMP(-1, done, "Could not locate pixel_mask");
+		fprintf(stderr, "WARNING: Could not find pixel mask - no masking will be applied\n");
 	}
 
 	/* determine where the data is stored and what strategy to use */
 	/* we select the "dectris-eiger" strategy if both are valid due to
-	 * potential confusion with the sizes of a virtual dataset (and possible
+	 * potential confusion with the sizes of a virtual dataset, possible failure
-	 * failure opening it if the library version is not up to date)
+	 * opening if we're using an old library version, and the potential to use the
+	 * optimised chunk read strategy
 	 */
 	if (H5Lexists(visit_result->nxdetector, "data_000001", H5P_DEFAULT) > 0) {
-		data_desc->data_group_id = visit_result->nxdetector;
+		ds_id = visit_result->nxdetector;
-		data_desc->get_data_properties = &get_dectris_eiger_dataset_dims;
+		ds_prop_func = &get_dectris_eiger_dataset_dims;
-		data_desc->get_data_frame = &get_dectris_eiger_frame;
+		frame_func = &get_dectris_eiger_frame;
 	} else if (H5Lexists(visit_result->nxdetector, "data", H5P_DEFAULT) > 0) {
-		data_desc->data_group_id = visit_result->nxdetector;
+		ds_id = visit_result->nxdetector;
-		data_desc->get_data_properties = &get_nxs_dataset_dims;
+		ds_prop_func = &get_nxs_dataset_dims;
-		data_desc->get_data_frame = &get_nxs_frame;
+		frame_func = &get_nxs_frame;
 	} else if (H5Lexists(visit_result->nxdata, "data_000001", H5P_DEFAULT) > 0) {
-		data_desc->data_group_id = visit_result->nxdata;
+		ds_id = visit_result->nxdata;
-		data_desc->get_data_properties = &get_dectris_eiger_dataset_dims;
+		ds_prop_func = &get_dectris_eiger_dataset_dims;
-		data_desc->get_data_frame = &get_dectris_eiger_frame;
+		frame_func = &get_dectris_eiger_frame;
 	} else if (H5Lexists(visit_result->nxdata, "data", H5P_DEFAULT) > 0) {
-		data_desc->data_group_id = visit_result->nxdata;
+		ds_id = visit_result->nxdata;
-		data_desc->get_data_properties = &get_nxs_dataset_dims;
+		ds_prop_func = &get_nxs_dataset_dims;
-		data_desc->get_data_frame = &get_nxs_frame;
+		frame_func = &get_nxs_frame;
 	} else {
-		data_desc->data_group_id = 0;
-		data_desc->get_data_properties = NULL;
-		data_desc->get_data_frame = NULL;
 		ERROR_JUMP(-1, done, "Could not locate detector dataset");
 	}
 
-	if (data_desc->get_data_properties == &get_dectris_eiger_dataset_dims) {
+
-		/* setup the "extra eiger info" struct */
+	if (ds_prop_func == &get_dectris_eiger_dataset_dims) {
-		struct eiger_data_description_t *eiger_desc = malloc(sizeof(*eiger_desc));
+
+		/* setup the "extra info" structs */
+		struct eiger_ds_desc_t *eiger_desc;
+		struct opt_eiger_ds_desc_t *o_eiger_desc;
+
+		eiger_desc = malloc(sizeof(*eiger_desc));
 		if (!eiger_desc) {
 			ERROR_JUMP(-1, done, "Memory error creating data description for Eiger");
 		}
 		memset(eiger_desc, 0, sizeof(*eiger_desc));
-		data_desc->extra = eiger_desc;
+		eiger_desc->frame_func = &get_frame_simple;
-		data_desc->free_extra = free_eiger_data_description;
+
-	} else {
+		o_eiger_desc = malloc(sizeof(*o_eiger_desc));
-		data_desc->free_extra = free_nxs_data_description;
+		if (!o_eiger_desc) {
+			free(eiger_desc);
+			ERROR_JUMP(-1, done, "Memory error creating data description for optimised Eiger");
 		}
+		o_eiger_desc->base.frame_func = &get_frame_from_chunk;
+
+		/* check if we can perform the optimised chunk read */
+		retval = check_for_chunk_read(ds_id, "data_000001", o_eiger_desc);
+		if (retval < 0) {
+			free(o_eiger_desc);
+			free(eiger_desc);
+			ERROR_JUMP(-1, done, "");
+		}
+		if (retval) {
+			free(eiger_desc);
+			*(struct opt_eiger_ds_desc_t**) desc = o_eiger_desc;
+			free_func = &free_opt_eiger_desc;
+		} else {
+			free(o_eiger_desc);
+			*(struct eiger_ds_desc_t**) desc = eiger_desc;
+			free_func = &free_eiger_desc;
+		}
+
+	} else {
+		*desc = malloc(sizeof(struct nxs_ds_desc_t));
+		free_func = &free_nxs_desc;
+	}
+
+	output = *((struct ds_desc_t **) desc);
+	output->det_g_id = g_id;
+	output->data_g_id = ds_id;
+	output->get_pixel_properties = pxl_func;
+	output->get_pixel_mask = pxl_mask_func;
+	output->get_data_frame = frame_func;
+	output->free_desc = free_func;
+
+	ds_prop_func(output);
 
 done:
 	return retval;
 }
 
 
-int extract_detector_info(
+int get_detector_info(
 		const hid_t fid,
-		struct data_description_t *data_desc,
+		struct ds_desc_t **desc) {
-		struct dataset_properties_t *dataset_prop) {
+
 	int retval = 0;
 	herr_t err = 0;
 	struct det_visit_objects_t objects = {0};
@@ -696,12 +921,11 @@ int extract_detector_info(
 		fprintf(stderr, "WARNING: Could not locate an NXdetector entry\n");
 	}
 
-	if ((retval = fill_data_descriptor(data_desc, &objects)) < 0) {
+	if ((retval = create_dataset_descriptor(desc, &objects)) < 0) {
 		ERROR_JUMP(retval, done, "");
 	};
-	if ((retval = data_desc->get_data_properties(data_desc, dataset_prop)) < 0) {
+
-		ERROR_JUMP(retval, done, "");
+
-	}
 done:
 	return retval;
 }

src/file.h

@@ -9,43 +9,42 @@
 
 #include <hdf5.h>
 #include "err.h"
+#include "filters.h"
 
-struct dataset_properties_t {
+
-	int data_width;
+struct ds_desc_t {
+	hid_t det_g_id;
+	hid_t data_g_id;
 	hsize_t dims[3];
+	int data_width;
+	int (*get_pixel_properties)(const struct ds_desc_t*, double*, double*);
+	int (*get_pixel_mask)(const struct ds_desc_t*, int*);
+	int (*get_data_frame)(const struct ds_desc_t*, const int, void*);
+	void (*free_desc)(struct ds_desc_t*);
 };
 
-struct data_description_t {
+struct nxs_ds_desc_t {
-	hid_t det_group_id;
+	struct ds_desc_t base;
-	hid_t data_group_id;
-	int (*get_pixel_properties)(const struct data_description_t*, double*, double*);
-	int (*get_pixel_mask)(const struct data_description_t*, int*);
-	int (*get_data_properties)(const struct data_description_t*, struct dataset_properties_t*);
-	int (*get_data_frame)(const struct data_description_t*, const struct dataset_properties_t*, int, int, void*);
-	void *extra;
-	void (*free_extra)(struct data_description_t*);
 };
 
-void free_nxs_data_description(struct data_description_t *desc);
+struct eiger_ds_desc_t {
-
+	struct ds_desc_t base;
-struct eiger_data_description_t {
 	int n_data_blocks;
 	int *block_sizes;
+	int (*frame_func)(const struct ds_desc_t*, const char*, const hsize_t*, const hsize_t*, void*);
 };
 
-void free_eiger_data_description(struct data_description_t *desc);
+struct opt_eiger_ds_desc_t {
+	struct eiger_ds_desc_t base;
+	int bs_applied;
+	unsigned int bs_params[BS_H5_N_PARAMS];
+};
+
+int get_detector_info(const hid_t fid, struct ds_desc_t **desc);
 
 struct det_visit_objects_t {
 	hid_t nxdata;
 	hid_t nxdetector;
 };
 
-void clear_det_visit_objects(struct det_visit_objects_t *objects);
-
-int get_nxs_dataset_dims(const struct data_description_t *desc, struct dataset_properties_t *properties);
-
-int fill_data_descriptor(struct data_description_t *data_desc, struct det_visit_objects_t *visit_result);
-
-int extract_detector_info(const hid_t fid, struct data_description_t *data_desc, struct dataset_properties_t *ds_prop);
-
 #endif /* NXS_XDS_FILE_H */

src/filters.c

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2018 Diamond Light Source Ltd.
+ * Author: Charles Mita
+ */
+
+#include <stdio.h>
+#include "filters.h"
+#include "err.h"
+#include "bitshuffle.h"
+
+/* Required prototypes from bitshuffle.c but not included in header */
+uint64_t bshuf_read_uint64_BE(const void *buffer);
+uint32_t bshuf_read_uint32_BE(const void *buffer);
+
+
+/*
+ * Derived from the h5 filter code from the bitshuffle project (not included here)
+ */
+int bslz4_decompress(
+		const unsigned int* bs_params,
+		size_t in_size,
+		void *in_buffer,
+		size_t out_size,
+		void *out_buffer) {
+
+	int retval = 0;
+	size_t size, elem_size, block_size, u_bytes;
+
+	elem_size = bs_params[2];
+	u_bytes = bshuf_read_uint64_BE(in_buffer);
+
+	if (u_bytes != out_size) {
+		char message[64];
+		sprintf(message, "Decompressed chunk is %lu bytes, expected %lu", u_bytes, out_size);
+		ERROR_JUMP(-1, done, message);
+	}
+
+	block_size = bshuf_read_uint32_BE((const char *) in_buffer + 8) / elem_size;
+	if (!block_size) {
+		ERROR_JUMP(-1, done, "Read block bitshuffle lz4 block size as 0");
+	}
+	/* skip over header */
+	in_buffer += 12;
+	size = u_bytes / elem_size;
+
+	if (bs_params[4] == BS_H5_PARAM_LZ4_COMPRESS) {
+		if (bshuf_decompress_lz4(in_buffer, out_buffer, size, elem_size, block_size) < 0) {
+			ERROR_JUMP(-1, done, "Error performing bitshuffle_lz4 decompression");
+		}
+	} else {
+		if (bshuf_bitunshuffle(in_buffer, out_buffer, size, elem_size, block_size) < 0) {
+			ERROR_JUMP(-1, done, "Error performing bit unshuffle");
+		}
+	}
+
+done:
+	return retval;
+}

src/filters.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2018 Diamond Light Source Ltd.
+ * Author: Charles Mita
+ */
+
+#ifndef NXS_XDS_FILTER_H
+#define NXS_XDS_FILTER_H
+
+#define BS_H5_N_PARAMS 5
+#define BS_H5_FILTER_ID 32008
+#define BS_H5_PARAM_LZ4_COMPRESS 2
+
+
+
+int bslz4_decompress(
+		const unsigned int* bs_params,
+		size_t in_size,
+		void *in_buffer,
+		size_t out_size,
+		void *out_buffer);
+
+#endif /* NXS_XDS_FILTER_H */

src/plugin.c

@@ -7,6 +7,7 @@
 #include <hdf5.h>
 #include <stdlib.h>
 #include "file.h"
+#include "filters.h"
 #include "plugin.h"
 
 
@@ -15,9 +16,40 @@
 #define ERROR_OUTPUT stderr
 
 
+
+/* mask bits loosely based on what Neggia does and what NeXus says should be done */
+/* basically - anything in the low byte (& 0xFF) means "ignore this" */
+/* Neggia usses the value -2 if bit 1, 2 or 3 are set */
+#define COPY_AND_MASK(in, out, size, mask) \
+{ \
+	int i; \
+	if (mask) { \
+		for (i = 0; i < size; ++i) { \
+			out[i] = in[i]; \
+			if (mask[i] & 0xFF) out[i] = -1; \
+			if (mask[i] & 30) out[i] = -2; \
+		} \
+	} else { \
+		for (i = 0; i < size; i++) { \
+			out[i] = in[i]; \
+		} \
+	} \
+}
+
+#define APPLY_MASK(buffer, mask, size) \
+{ \
+	int i; \
+	if (mask) { \
+		for (i = 0; i < size; ++i) { \
+			if (mask[i] & 0xFF) buffer[i] = -1; \
+			if (mask[i] & 30) buffer[i] = -2; \
+		} \
+	} \
+}
+
+
 static hid_t file_id = 0;
-static struct data_description_t data_desc = {0};
+static struct ds_desc_t *data_desc = NULL;
-static struct dataset_properties_t ds_prop = {0};
 static int *mask_buffer = NULL;
 
 
@@ -29,21 +61,35 @@ void fill_info_array(int info[1024]) {
 	info[4] = VERSION_TIMESTAMP;
 }
 
-void apply_mask(int *data, int *mask, int size) {
+int convert_to_int_and_mask(void *in_buffer, int d_width, int *out_buffer, int length, int *mask) {
-	int *dptr, *mptr;
+	/* transfer data to output buffer, performing data conversion as required */
-	dptr = data;
+	int retval = 0;
-	mptr = mask;
+	/* TODO: decide how conversion of data should work */
-	while (dptr < data + size && mptr < mask + size) {
+	/* Should we sign extend? Neggia doesn't (casts from uint*), but may be more intuitive */
-		/* mask bits loosely based on what Neggia does and what NeXus says should be done */
+	if (d_width == sizeof(signed char)) {
-		/* basically - anything in the low byte (& 0xFF) means "ignore this" */
+		signed char *in = in_buffer;
-		if (*mptr & 0x01) *dptr = -1;
+		COPY_AND_MASK(in, out_buffer, length, mask);
-		if (*mptr & 0xFE) *dptr = -2;
+	} else if (d_width == sizeof(short)) {
-		dptr++;
+		short *in = in_buffer;
-		mptr++;
+		COPY_AND_MASK(in, out_buffer, length, mask);
+	} else if (d_width == sizeof(int)) {
+		int *in = in_buffer;
+		COPY_AND_MASK(in, out_buffer, length, mask);
+	} else if (d_width == sizeof(long int)) {
+		long int *in = in_buffer;
+		COPY_AND_MASK(in, out_buffer, length, mask);
+	} else if (d_width == sizeof(long long int)) {
+		long long int *in = in_buffer;
+		COPY_AND_MASK(in, out_buffer, length, mask);
+	} else {
+		char message[128];
+		sprintf(message, "Unsupported conversion of data width %d to %ld (int)", d_width, sizeof(int));
+		ERROR_JUMP(-1, done, message);
 	}
+done:
+	return retval;
 }
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -74,14 +120,14 @@ void plugin_open(
 	}
 
 	reset_error_stack();
-	retval = extract_detector_info(file_id, &data_desc, &ds_prop);
+	retval = get_detector_info(file_id, &data_desc);
 	if (retval < 0) {
 		ERROR_JUMP(-4, done, "");
 	}
 
-	mask_buffer = malloc(ds_prop.dims[1] * ds_prop.dims[2] * sizeof(int));
+	mask_buffer = malloc(data_desc->dims[1] * data_desc->dims[2] * sizeof(int));
 	if (mask_buffer) {
-		retval = data_desc.get_pixel_mask(&data_desc, mask_buffer);
+		retval = data_desc->get_pixel_mask(data_desc, mask_buffer);
 		if (retval < 0) {
 			fprintf(ERROR_OUTPUT, "WARNING: Could not read pixel mask - no masking will be applied\n");
 			dump_error_stack(ERROR_OUTPUT);
@@ -94,6 +140,10 @@ void plugin_open(
 done:
 	*error_flag = retval;
 	if (retval < 0) {
+		if ((data_desc) && (data_desc->free_desc)) {
+			data_desc->free_desc(data_desc);
+			data_desc = NULL;
+		}
 		dump_error_stack(ERROR_OUTPUT);
 	}
 }
@@ -112,15 +162,15 @@ void plugin_get_header(
 	reset_error_stack();
 	fill_info_array(info);
 
-	err = data_desc.get_pixel_properties(&data_desc, &x_pixel_size, &y_pixel_size);
+	err = data_desc->get_pixel_properties(data_desc, &x_pixel_size, &y_pixel_size);
 	if (err < 0) {
 		ERROR_JUMP(err, done, "Failed to retrieve pixel information");
 	}
 
-	*nx = ds_prop.dims[2];
+	*nx = data_desc->dims[2];
-	*ny = ds_prop.dims[1];
+	*ny = data_desc->dims[1];
-	*nbytes = ds_prop.dims[1] * ds_prop.dims[2] * ds_prop.data_width;
+	*nbytes = data_desc->data_width;
-	*number_of_frames = ds_prop.dims[0];
+	*number_of_frames = data_desc->dims[0];
 	*qx = (float) x_pixel_size;
 	*qy = (float) y_pixel_size;
 
@@ -138,16 +188,36 @@ void plugin_get_data(
 		int *data_array,
 		int info[1024],
 		int *error_flag) {
+
 	int retval = 0;
+	int frame_size_px = data_desc->dims[1] * data_desc->dims[2];
 	reset_error_stack();
 	fill_info_array(info);
-	if (data_desc.get_data_frame(&data_desc, &ds_prop, (*frame_number) - 1, sizeof(int), data_array) < 0) {
+
+	void *buffer = NULL;
+	if (sizeof(*data_array) == data_desc->data_width) {
+		buffer = data_array;
+	} else {
+		buffer = malloc(data_desc->data_width * frame_size_px);
+		if (!buffer) {
+			ERROR_JUMP(-1, done, "Unable to allocate data buffer");
+		}
+	}
+
+	if (data_desc->get_data_frame(data_desc, (*frame_number) - 1, buffer) < 0) {
 		char message[64] = {0};
 		sprintf(message, "Failed to retrieve data for frame %d", *frame_number);
 		ERROR_JUMP(-2, done, message);
 	}
-	if (mask_buffer) {
+
-		apply_mask(data_array, mask_buffer, ds_prop.dims[1] * ds_prop.dims[2]);
+	if (buffer != data_array) {
+		if (convert_to_int_and_mask(buffer, data_desc->data_width, data_array, frame_size_px, mask_buffer) < 0) {
+			char message[64];
+			sprintf(message, "Error converting data for frame %d", *frame_number);
+			ERROR_JUMP(-2, done, message);
+		}
+	} else {
+		APPLY_MASK(data_array, mask_buffer, frame_size_px);
 	}
 
 done:
@@ -155,6 +225,7 @@ done:
 	if (retval < 0) {
 		dump_error_stack(ERROR_OUTPUT);
 	}
+	if (buffer && (buffer != data_array)) free(buffer);
 }
 
 
@@ -168,7 +239,10 @@ void plugin_close(int *error_flag) {
 	file_id = 0;
 
 	if (mask_buffer) free(mask_buffer);
-	if (data_desc.free_extra) data_desc.free_extra(&data_desc);
+	if (data_desc->free_desc) {
+		data_desc->free_desc(data_desc);
+		data_desc = NULL;
+	}
 	if (H5close() < 0) {
 		*error_flag = -1;
 	}

src/test.c

@@ -4,19 +4,21 @@
 #include "file.h"
 #include "err.h"
 
-
+#define COPY_AND_MASK(in, out, size, mask) \
-void apply_mask(int *data, int *mask, int size) {
+{ \
-	int *dptr, *mptr;
+	int i; \
-	dptr = data;
+	if (mask) { \
-	mptr = mask;
+		for (i = 0; i < size; ++i) { \
-	while (dptr < data + size && mptr < mask + size) {
+			out[i] = in[i]; \
-		if (*mptr & 0x01) *dptr = -1;
+			if (mask[i] & 0xFE) out[i] = -2; \
-		if (*mptr & 0xFE) *dptr = -2;
+			if (mask[i] & 0x01) out[i] = -1; \
-		dptr++;
+		} \
-		mptr++;
+	} else { \
+		for (i = 0; i < size; i++) { \
+			out[i] = in[i]; \
+		} \
+	} \
 }
-}
-
 
 int parse_args(int argc, char **argv, char **file_name, int *frame_idx) {
 	int retval = 0;
@@ -38,13 +40,13 @@ int main(int argc, char **argv) {
 	int err = 0;
 	int retval = 0;
 	char *test_file = "";
-	struct data_description_t desc = {0};
+	struct ds_desc_t *desc;
-	struct dataset_properties_t prop = {0};
 	int dims[3] = {0};
 	hid_t fid = 0;
 	int frame_idx = 0;
 	int *mask = NULL;
 	int *data = NULL;
+	void *buffer = NULL;
 
 	init_error_handling();
 	if (init_h5_error_handling() < 0) {
@@ -58,13 +60,13 @@ int main(int argc, char **argv) {
 	fid = H5Fopen(test_file, H5F_ACC_RDONLY, H5P_DEFAULT);
 	if (fid < 0) ERROR_JUMP(-1, done, "Error opening file");
 
-	err = extract_detector_info(fid, &desc, &prop);
+	err = get_detector_info(fid, &desc);
 	if (err < 0) {
 		ERROR_JUMP(err, done, "");
 	}
-	dims[0] = prop.dims[0];
+	dims[0] = desc->dims[0];
-	dims[1] = prop.dims[1];
+	dims[1] = desc->dims[1];
-	dims[2] = prop.dims[2];
+	dims[2] = desc->dims[2];
 
 	printf("Dims: %d, %d, %d\n", dims[0], dims[1], dims[2]);
 
@@ -72,18 +74,42 @@ int main(int argc, char **argv) {
 	if (!mask) {
 		ERROR_JUMP(err, done, "Failed to allocate space for pixel mask");
 	}
-	err = desc.get_pixel_mask(&desc, mask);
+	err = desc->get_pixel_mask(desc, mask);
 	if (err < 0) {
 		ERROR_JUMP(err, done, "");
 	}
 
 	data = malloc(dims[1] * dims[2] * sizeof(*data));
-	err = desc.get_data_frame(&desc, &prop, frame_idx, sizeof(*data), data);
+	if (sizeof(*data) != desc->data_width) {
+		buffer = malloc(dims[1] * dims[2] * desc->data_width);
+	} else {
+		buffer = data;
+	}
+
+	err = desc->get_data_frame(desc, frame_idx, buffer);
 	if (err < 0) {
 		ERROR_JUMP(err, done, "");
 	}
 
-	apply_mask(data, mask, dims[1] * dims[2]);
+	if (buffer != data) {
+		if (desc->data_width == sizeof(signed char)) {
+			signed char *in = buffer;
+			COPY_AND_MASK(in, data, dims[1] * dims[2], mask);
+		} else if (desc->data_width == sizeof(short)) {
+			short *in = buffer;
+			COPY_AND_MASK(in, data, dims[1] * dims[2], mask);
+		} else if (desc->data_width == sizeof(int)) {
+			int *in = buffer;
+			COPY_AND_MASK(in, data, dims[1] * dims[2], mask);
+		} else if (desc->data_width == sizeof(long int)) {
+			long int *in = buffer;
+			COPY_AND_MASK(in, data, dims[1] * dims[2], mask);
+		} else if (desc->data_width == sizeof(long long int)) {
+			long long int *in = buffer;
+			COPY_AND_MASK(in, data, dims[1] * dims[2], mask);
+		}
+	}
+
 	{
 		int i, j;
 		int max_i = 30;
@@ -101,6 +127,7 @@ int main(int argc, char **argv) {
 done:
 	if (fid > 0) H5Fclose(fid);
 	if (data) free(data);
+	if (buffer && (data != buffer)) free(buffer);
 	if (mask) free(mask);
 	if (retval != 0) dump_error_stack(stderr);
 	return retval;