From 4cf18ceff4f8395ebfeaed927d7f30cdb2c7d461 Mon Sep 17 00:00:00 2001
From: Clemens Vonrhein <vonrhein@globalphasing.com>
Date: Mon, 16 Dec 2024 15:05:38 +0000
Subject: [PATCH] Mechanism to read bitshuffle-compressed pixel-mask data - by
 loading/using the normal bitshuffle source from

  https://github.com/kiyo-masui/bitshuffle

(see Makefile).
---
 Makefile     | 20 ++++++++++++++++++--
 src/file.c   | 26 +++++++++++++++++++++++++-
 src/plugin.c | 10 ++++++++++
 3 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 4387348..0a6b257 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,19 @@ BSLZ4_BUILD_DIR = ./bslz4/build
 BSLZ4_INC_DIR = $(BSLZ4_SRC_DIR)
 
 CC=h5cc
-CFLAGS=-DH5_USE_110_API -Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=c99 -shlib
+# -std=gnu99 provides for strtok_r
+CFLAGS=-DH5_USE_110_API -Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=gnu99 -shlib
+
+# include https://github.com/kiyo-masui/bitshuffle (to handle
+# e.g. bitshuffle compressed pixel_masks)
+use_BITSHUFFLE =
+ifeq ($(use_BITSHUFFLE),)
+else
+BITSHUFFLE_SRC_DIR = ../bitshuffle-master/src/
+BITSHUFFLE_INC_DIR = ../bitshuffle-master/src/
+BITSHUFFLE_OBJS    = $(BUILD_DIR)/bshuf_h5filter.o
+CFLAGS += -DUSE_BITSHUFFLE -I$(BITSHUFFLE_INC_DIR)
+endif
 
 .PHONY: plugin
 plugin: $(BUILD_DIR)/durin-plugin.so
@@ -26,6 +38,10 @@ $(BUILD_DIR)/test_plugin: $(TEST_DIR)/generic_data_plugin.f90 $(TEST_DIR)/test_g
 	mkdir -p $(BUILD_DIR)
 	gfortran -O -g -fopenmp -ldl $(TEST_DIR)/generic_data_plugin.f90 $(TEST_DIR)/test_generic_host.f90 -o $@ -J$(BUILD_DIR)
 
+$(BUILD_DIR)/bshuf_h5filter.o: $(BITSHUFFLE_SRC_DIR)/bshuf_h5filter.c
+	mkdir -p $(BUILD_DIR)
+	$(CC) $(CFLAGS) -c $< -o $@
+
 $(BUILD_DIR)/%.o: $(SRC_DIR)/%.c
 	mkdir -p $(BUILD_DIR)
 	$(CC) $(CFLAGS) -c $< -o $@
@@ -39,7 +55,7 @@ $(BSLZ4_BUILD_DIR)/bitshuffle_core.o $(BSLZ4_BUILD_DIR)/iochain.o
 	mkdir -p $(BUILD_DIR)
 	ar rcs $@ $^
 
-$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o \
+$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o $(BITSHUFFLE_OBJS) \
 $(BUILD_DIR)/bslz4.a
 	mkdir -p $(BUILD_DIR)
 	$(CC) $(CFLAGS) -shared -noshlib $^ -o $(BUILD_DIR)/durin-plugin.so
diff --git a/src/file.c b/src/file.c
index 7a6017d..85bf2f8 100644
--- a/src/file.c
+++ b/src/file.c
@@ -573,9 +573,33 @@ int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) {
     ERROR_JUMP(-1, done, "Error opening detectorSpecific/pixel_mask");
   }
 
+  // what if this is compressed?
+  hid_t dcpl    = H5Dget_create_plist(ds_id);
+  int n_filters = H5Pget_nfilters(dcpl);
+  H5Z_filter_t    filter_id;
+  if (n_filters>0) {
+	unsigned int    flags;
+	size_t          nelmts = 1;
+	unsigned int    values_out[1] = {99};
+	char            filter_name[80];
+	filter_id = H5Pget_filter(dcpl, (unsigned) 0, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL);
+	if (filter_id>=0) {
+	  fprintf(stderr," filter name =\"%s\"\n",filter_name);
+	}
+  }
+
   err = H5Dread(ds_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer);
   if (err < 0) {
-    ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask");
+	if (n_filters>0) {
+	  ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask with filter(s)");
+	}
+	else {
+	  ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask");
+	}
+  }
+
+  if (H5Zfilter_avail(BS_H5_FILTER_ID)) {
+	fprintf(stderr," bitshuffle filter is available now since H5Dread (of pixel-mask) triggered loading of the filter.\n");
   }
 
 close_dataset:
diff --git a/src/plugin.c b/src/plugin.c
index 0f5c3e9..9c2e3b3 100644
--- a/src/plugin.c
+++ b/src/plugin.c
@@ -11,6 +11,10 @@
 #include "filters.h"
 #include "plugin.h"
 
+#ifdef USE_BITSHUFFLE
+#include "bshuf_h5filter.h"
+#endif
+
 /* XDS does not provide an error callback facility, so just write to stderr
    for now - generally regarded as poor practice */
 #define ERROR_OUTPUT stderr
@@ -286,6 +290,12 @@ void plugin_open(const char *filename, int info[1024], int *error_flag) {
     ERROR_JUMP(-2, done, "Failed to configure HDF5 error handling");
   }
 
+#ifdef USE_BITSHUFFLE
+  if (bshuf_register_h5filter() < 0 ) {
+    ERROR_JUMP(-2, done, "Failed to register bitshuffle filter");
+  }
+#endif
+
   fill_info_array(info);
   file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
   if (file_id < 0) {