From 53a91643a227e4fdb20321d000bcd0ce59bfdcf3 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 7 Dec 2018 14:06:50 +0100
Subject: [PATCH 01/19] initial commit

---
 .gitignore                  |  4 ++
 Readme.md                   | 29 ++++++++++++
 cadump/__init__.py          |  0
 cadump/cadump.py            | 88 +++++++++++++++++++++++++++++++++++++
 conda-recipe/meta.yaml      | 17 +++++++
 setup.py                    | 25 +++++++++++
 tests/__init__.py           |  0
 tests/channels.txt          |  5 +++
 tests/test_download_data.py | 35 +++++++++++++++
 9 files changed, 203 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Readme.md
 create mode 100644 cadump/__init__.py
 create mode 100644 cadump/cadump.py
 create mode 100644 conda-recipe/meta.yaml
 create mode 100644 setup.py
 create mode 100644 tests/__init__.py
 create mode 100644 tests/channels.txt
 create mode 100644 tests/test_download_data.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..77d9edc
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+__pycache__
+.idea
+.pytest_cache
+*.h5
diff --git a/Readme.md b/Readme.md
new file mode 100644
index 0000000..4d96c6f
--- /dev/null
+++ b/Readme.md
@@ -0,0 +1,29 @@
+# Overview
+Simple server to dump Epics Channel Access data to an HDF5 file.
+The server gets an http callback from the Broker whenever there was an acquisition.
+
+The format of the request is as follows:
+```
+{
+	'range': {
+		'startPulseId': 100, 
+	 	'endPulseId': 120
+	}, 
+		
+	'parameters': {
+		'general/created': 'test', 
+		'general/user': 'tester', 
+		'general/process': 'test_process', 
+		'general/instrument': 'mac', 
+		'output_file': '/bla/test.h5'}
+}
+
+```
+
+Right now this server needs to run on the same server than the
+
+# Testing
+
+```bash
+curl -XPUT -d '{"range":{"startPulseId": 7281433214, "endPulseId": 7281489688}, "parameters":{"output_file":"test.h5"}}' http://localhost:10200/notify
+``` 
\ No newline at end of file
diff --git a/cadump/__init__.py b/cadump/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/cadump/cadump.py b/cadump/cadump.py
new file mode 100644
index 0000000..57f0075
--- /dev/null
+++ b/cadump/cadump.py
@@ -0,0 +1,88 @@
+from bottle import route, run, request, abort
+import json
+import logging
+
+import data_api
+
+# This is how the notification look like
+# {
+#     'range': {
+#         'startPulseId': 100,
+#         'endPulseId': 120
+#     },
+#
+#     'parameters': {
+#         'general/created': 'test',
+#         'general/user': 'tester',
+#         'general/process': 'test_process',
+#         'general/instrument': 'mac',
+#         'output_file': '/bla/test.h5'}    # this is usually the full path
+# }
+
+channel_list = ["S10-CPCL-VM1MGC:LOAD"]  # specified channel is only for test purposes
+
+
+@route('/notify', method='PUT')
+def put_document():
+    data = request.body.read()
+    if not data:
+        abort(400, 'No data received')
+
+    try:
+        download_data(json.loads(data))
+    except Exception as e:
+        logging.warning("Download data failed", e)
+
+
+def download_data(config):
+
+    logging.info("Dump data to hdf5 ...")
+    # logging.info(config)
+
+    start_pulse = config["range"]["startPulseId"]
+    end_pulse = config["range"]["endPulseId"]
+
+    start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
+
+    print(start_date, end_date)
+
+    # append _CA to the filename
+    filename = config["parameters"]["output_file"]
+    new_filename = filename[:-3]+"_CA"+filename[-3:]
+
+    logging.info("Retrieving data")
+    data = data_api.get_data(channel_list, start=start_date, end= end_date)
+    logging.info("Persist data to hdf5 file")
+    data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
+
+
+def read_channels(filename):
+    with open(filename) as f:
+        lines = f.readlines()
+
+    channels = []
+    for line in lines:
+        line = line.strip()
+        if line: # if not empty line
+            channels.append(line) # remove all leading and trailing spaces
+
+    return channels
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description='Channel Access archiver dump to hdf5')
+    parser.add_argument('--channels', dest='channel_list', default="tests/channels.txt", help='channels to dump')
+
+    args = parser.parse_args()
+    print(args.channel_list)
+
+    global channel_list
+    channel_list = read_channels(args.channel_list)
+    logging.info("Using channel list: " + " ".join(channel_list))
+
+    run(host='localhost', port=10200)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
new file mode 100644
index 0000000..18ab23d
--- /dev/null
+++ b/conda-recipe/meta.yaml
@@ -0,0 +1,17 @@
+{% set data = load_setup_py_data() %}
+
+package:
+  name: cadump
+  version: {{ data.get('version') }}
+
+build:
+  entry_points:
+    - cadump_server = cadump.cadump:main
+
+requirements:
+  build:
+    - python
+    - data_api
+  run:
+    - python
+    - data_api
\ No newline at end of file
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..62d151a
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,25 @@
+import os
+from setuptools import setup, find_packages
+
+
+# Utility function to read the README file.
+# Used for the long_description.  It's nice, because now 1) we have a top level
+# README file and 2) it's easier to type in the README file than to put a raw
+# string in below ...
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+setup(
+    name="cadump",
+    version="0.0.1",
+    author="Paul Scherrer Institute",
+    author_email="daq@psi.ch",
+    description=("Interface to dump data from archiver/databuffer"),
+    license="GPLv3",
+    keywords="",
+    url="",
+    packages=find_packages(),
+    long_description=read('Readme.md'),
+
+)
\ No newline at end of file
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/channels.txt b/tests/channels.txt
new file mode 100644
index 0000000..7033f9e
--- /dev/null
+++ b/tests/channels.txt
@@ -0,0 +1,5 @@
+S10-CPCL-VM1MGC:LOAD
+
+ONE
+ TWO
+THREE
diff --git a/tests/test_download_data.py b/tests/test_download_data.py
new file mode 100644
index 0000000..7a9d968
--- /dev/null
+++ b/tests/test_download_data.py
@@ -0,0 +1,35 @@
+import unittest
+from unittest import TestCase
+from cadump import cadump
+import logging
+
+class TestDownloadData(TestCase):
+
+    def test_download_data(self):
+        config = {
+            'range': {
+                'startPulseId': 7281433214,
+                'endPulseId': 7281489688
+            },
+
+            'parameters': {
+                'general/created': 'test',
+                'general/user': 'tester',
+                'general/process': 'test_process',
+                'general/instrument': 'mac',
+                'output_file': 'test.h5'}    # this is usually the full path
+        }
+
+        cadump.download_data(config)
+        # self.fail()
+
+    def test_read_channels(self):
+        channels = cadump.read_channels("channels.txt")
+        logging.info(channels)
+        self.assertEqual(len(channels), 3)
+
+
+
+
+if __name__ == '__main__':
+    unittest.main()

From 703d8ce88350004de330527b1a1cff794814f377 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 7 Dec 2018 14:14:23 +0100
Subject: [PATCH 02/19] fix logging

---
 cadump/cadump.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 57f0075..c2e0ad9 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -1,9 +1,11 @@
 from bottle import route, run, request, abort
 import json
-import logging
 
 import data_api
 
+import logging
+logger = logging.getLogger("logger");
+
 # This is how the notification look like
 # {
 #     'range': {
@@ -31,28 +33,26 @@ def put_document():
     try:
         download_data(json.loads(data))
     except Exception as e:
-        logging.warning("Download data failed", e)
+        logger.warning("Download data failed", e)
 
 
 def download_data(config):
 
-    logging.info("Dump data to hdf5 ...")
-    # logging.info(config)
+    logger.info("Dump data to hdf5 ...")
+    # logger.info(config)
 
     start_pulse = config["range"]["startPulseId"]
     end_pulse = config["range"]["endPulseId"]
 
     start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
 
-    print(start_date, end_date)
-
     # append _CA to the filename
     filename = config["parameters"]["output_file"]
     new_filename = filename[:-3]+"_CA"+filename[-3:]
 
-    logging.info("Retrieving data")
+    logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date))
     data = data_api.get_data(channel_list, start=start_date, end= end_date)
-    logging.info("Persist data to hdf5 file")
+    logger.info("Persist data to hdf5 file")
     data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
 
 
@@ -79,7 +79,7 @@ def main():
 
     global channel_list
     channel_list = read_channels(args.channel_list)
-    logging.info("Using channel list: " + " ".join(channel_list))
+    logger.info("Using channel list: " + " ".join(channel_list))
 
     run(host='localhost', port=10200)
 

From 1fe28f71f7aca1178928a2d25a97516790130215 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 7 Dec 2018 14:20:07 +0100
Subject: [PATCH 03/19] fix version - as jinja templating not yet supported on
 my python

---
 conda-recipe/meta.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 18ab23d..4a68149 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,8 +1,6 @@
-{% set data = load_setup_py_data() %}
-
 package:
   name: cadump
-  version: {{ data.get('version') }}
+  version: 0.0.1
 
 build:
   entry_points:

From eed0bc469fbd09338b28f534af90ae950c0de662 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 7 Dec 2018 14:54:38 +0100
Subject: [PATCH 04/19] updates

---
 conda-recipe/build.sh  |  3 +++
 conda-recipe/meta.yaml |  5 ++++-
 setup.py               | 23 ++++-------------------
 3 files changed, 11 insertions(+), 20 deletions(-)
 create mode 100644 conda-recipe/build.sh

diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh
new file mode 100644
index 0000000..d7a34f9
--- /dev/null
+++ b/conda-recipe/build.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+$PYTHON setup.py install     # Python command to install the script
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 4a68149..3f7f405 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,9 @@
 package:
   name: cadump
-  version: 0.0.1
+  version: 0.0.2
+
+source:
+    path: ..
 
 build:
   entry_points:
diff --git a/setup.py b/setup.py
index 62d151a..800aafc 100644
--- a/setup.py
+++ b/setup.py
@@ -1,25 +1,10 @@
-import os
-from setuptools import setup, find_packages
-
-
-# Utility function to read the README file.
-# Used for the long_description.  It's nice, because now 1) we have a top level
-# README file and 2) it's easier to type in the README file than to put a raw
-# string in below ...
-def read(fname):
-    return open(os.path.join(os.path.dirname(__file__), fname)).read()
-
+from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.1",
+    version="0.0.2",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
-    description=("Interface to dump data from archiver/databuffer"),
-    license="GPLv3",
-    keywords="",
-    url="",
-    packages=find_packages(),
-    long_description=read('Readme.md'),
-
+    description="Interface to dump data from archiver/databuffer",
+    packages=["cadump"]
 )
\ No newline at end of file

From d3ef039b6bac3263d4e6fdd9faff40c6d21734b0 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 7 Dec 2018 15:23:05 +0100
Subject: [PATCH 05/19] add url option

---
 cadump/cadump.py       | 8 +++++++-
 conda-recipe/meta.yaml | 2 +-
 setup.py               | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index c2e0ad9..ab6df18 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -22,6 +22,7 @@ logger = logging.getLogger("logger");
 # }
 
 channel_list = ["S10-CPCL-VM1MGC:LOAD"]  # specified channel is only for test purposes
+base_url = ""
 
 
 @route('/notify', method='PUT')
@@ -51,7 +52,7 @@ def download_data(config):
     new_filename = filename[:-3]+"_CA"+filename[-3:]
 
     logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date))
-    data = data_api.get_data(channel_list, start=start_date, end= end_date)
+    data = data_api.get_data(channel_list, start=start_date, end= end_date, base_url=base_url)
     logger.info("Persist data to hdf5 file")
     data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
 
@@ -73,6 +74,7 @@ def main():
     import argparse
     parser = argparse.ArgumentParser(description='Channel Access archiver dump to hdf5')
     parser.add_argument('--channels', dest='channel_list', default="tests/channels.txt", help='channels to dump')
+    parser.add_argument('--url', dest='url', default=None, help='base url to retrieve data from')
 
     args = parser.parse_args()
     print(args.channel_list)
@@ -81,6 +83,10 @@ def main():
     channel_list = read_channels(args.channel_list)
     logger.info("Using channel list: " + " ".join(channel_list))
 
+    global base_url
+    base_url = args.url
+    logger.info("Using base url: " + str(base_url))
+
     run(host='localhost', port=10200)
 
 
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 3f7f405..0116fa9 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.2
+  version: 0.0.3
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 800aafc..bbfffe3 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.2",
+    version="0.0.3",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",

From 572958f1498744d19afecccd2a4be51cd05a706f Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Tue, 11 Dec 2018 09:08:45 +0100
Subject: [PATCH 06/19] add hack to more robustly get the pulseid-date mapping

---
 cadump/cadump.py            | 46 +++++++++++++++++++++++++++++++++----
 conda-recipe/meta.yaml      |  2 +-
 setup.py                    |  2 +-
 tests/test_download_data.py |  5 ++--
 4 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index ab6df18..7677731 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -2,9 +2,12 @@ from bottle import route, run, request, abort
 import json
 
 import data_api
+import requests
+import dateutil.parser
+import pytz
 
 import logging
-logger = logging.getLogger("logger");
+logger = logging.getLogger("logger")
 
 # This is how the notification look like
 # {
@@ -34,7 +37,7 @@ def put_document():
     try:
         download_data(json.loads(data))
     except Exception as e:
-        logger.warning("Download data failed", e)
+        logger.exception("Download data failed")
 
 
 def download_data(config):
@@ -45,14 +48,15 @@ def download_data(config):
     start_pulse = config["range"]["startPulseId"]
     end_pulse = config["range"]["endPulseId"]
 
-    start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
+    # start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
+    start_date, end_date = get_pulse_id_date_mapping([start_pulse, end_pulse])
 
     # append _CA to the filename
     filename = config["parameters"]["output_file"]
     new_filename = filename[:-3]+"_CA"+filename[-3:]
 
     logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date))
-    data = data_api.get_data(channel_list, start=start_date, end= end_date, base_url=base_url)
+    data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
     logger.info("Persist data to hdf5 file")
     data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
 
@@ -64,12 +68,44 @@ def read_channels(filename):
     channels = []
     for line in lines:
         line = line.strip()
-        if line: # if not empty line
+        if line:  # if not empty line
             channels.append(line) # remove all leading and trailing spaces
 
     return channels
 
 
+def get_pulse_id_date_mapping(pulse_ids):
+    try:
+        dates = []
+        for pulse_id in pulse_ids:
+
+            query = {"range": {"startPulseId": pulse_id,"endPulseId": 9223372036854775807},
+                     "limit": 1,
+                     "ordering": "asc",
+                     "channels": ["SIN-CVME-TIFGUN-EVR0:BEAMOK"],
+                     "fields": ["pulseId", "globalDate"]}
+
+            # Query server
+            response = requests.post("https://data-api.psi.ch/sf/query", json=query)
+
+            # Check for successful return of data
+            if response.status_code != 200:
+                raise RuntimeError("Unable to retrieve data from server: ", response)
+
+            data = response.json()
+
+            if not pulse_id == data[0]["data"][0]["pulseId"]:
+                raise RuntimeError('Unable to retrieve mapping')
+
+            date = data[0]["data"][0]["globalDate"]
+            date = dateutil.parser.parse(date)
+            dates.append(date)
+
+        return dates
+    except Exception:
+        raise RuntimeError('Unable to retrieve mapping')
+
+
 def main():
     import argparse
     parser = argparse.ArgumentParser(description='Channel Access archiver dump to hdf5')
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 0116fa9..85b3bd5 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.3
+  version: 0.0.4
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index bbfffe3..27772a3 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.3",
+    version="0.0.4",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",
diff --git a/tests/test_download_data.py b/tests/test_download_data.py
index 7a9d968..e5e30e1 100644
--- a/tests/test_download_data.py
+++ b/tests/test_download_data.py
@@ -8,8 +8,8 @@ class TestDownloadData(TestCase):
     def test_download_data(self):
         config = {
             'range': {
-                'startPulseId': 7281433214,
-                'endPulseId': 7281489688
+                'startPulseId': 7314794868,
+                'endPulseId': 7314798868
             },
 
             'parameters': {
@@ -20,6 +20,7 @@ class TestDownloadData(TestCase):
                 'output_file': 'test.h5'}    # this is usually the full path
         }
 
+        cadump.base_url = "https://data-api.psi.ch/sf"
         cadump.download_data(config)
         # self.fail()
 

From f61c41233e4cd283922ef1855202cde6375e9db6 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Tue, 11 Dec 2018 09:14:12 +0100
Subject: [PATCH 07/19] add reference

---
 cadump/cadump.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 7677731..55a0da2 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -75,6 +75,9 @@ def read_channels(filename):
 
 
 def get_pulse_id_date_mapping(pulse_ids):
+
+    # See https://jira.psi.ch/browse/ATEST-897 for more details ...
+    
     try:
         dates = []
         for pulse_id in pulse_ids:

From bbb98fe579e0c752958172cc4663de40dec57cb3 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Tue, 11 Dec 2018 15:00:57 +0100
Subject: [PATCH 08/19] fix delay download

---
 cadump/cadump.py            | 48 ++++++++++++++++++++++++++-----------
 conda-recipe/meta.yaml      |  2 +-
 setup.py                    |  2 +-
 tests/test_download_data.py |  2 +-
 4 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 55a0da2..075590e 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -5,6 +5,8 @@ import data_api
 import requests
 import dateutil.parser
 import pytz
+import datetime
+import time
 
 import logging
 logger = logging.getLogger("logger")
@@ -48,6 +50,7 @@ def download_data(config):
     start_pulse = config["range"]["startPulseId"]
     end_pulse = config["range"]["endPulseId"]
 
+    logger.info("Retrieve pulse-id / data mapping for pulse ids")
     # start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
     start_date, end_date = get_pulse_id_date_mapping([start_pulse, end_pulse])
 
@@ -77,32 +80,49 @@ def read_channels(filename):
 def get_pulse_id_date_mapping(pulse_ids):
 
     # See https://jira.psi.ch/browse/ATEST-897 for more details ...
-    
+
     try:
         dates = []
         for pulse_id in pulse_ids:
 
-            query = {"range": {"startPulseId": pulse_id,"endPulseId": 9223372036854775807},
+            query = {"range": {"startPulseId": 0,"endPulseId": pulse_id},
                      "limit": 1,
-                     "ordering": "asc",
+                     "ordering": "desc",
                      "channels": ["SIN-CVME-TIFGUN-EVR0:BEAMOK"],
                      "fields": ["pulseId", "globalDate"]}
 
-            # Query server
-            response = requests.post("https://data-api.psi.ch/sf/query", json=query)
+            for c in range(1):
+                # Query server
+                response = requests.post("https://data-api.psi.ch/sf/query", json=query)
 
-            # Check for successful return of data
-            if response.status_code != 200:
-                raise RuntimeError("Unable to retrieve data from server: ", response)
+                # Check for successful return of data
+                if response.status_code != 200:
+                    raise RuntimeError("Unable to retrieve data from server: ", response)
 
-            data = response.json()
+                data = response.json()
 
-            if not pulse_id == data[0]["data"][0]["pulseId"]:
-                raise RuntimeError('Unable to retrieve mapping')
+                if not pulse_id == data[0]["data"][0]["pulseId"]:
+                    if c == 0:
+                        ref_date = data[0]["data"][0]["globalDate"]
+                        ref_date = dateutil.parser.parse(ref_date)
 
-            date = data[0]["data"][0]["globalDate"]
-            date = dateutil.parser.parse(date)
-            dates.append(date)
+                        now_date = datetime.datetime.now()
+                        now_date = pytz.timezone('Europe/Zurich').localize(now_date)
+
+                        check_date = ref_date+datetime.timedelta(seconds=20)
+                        delta_date = check_date - now_date
+
+                        s = delta_date.seconds
+                        logger.info("retry in " + str(s) + " seconds ")
+                        if not s <= 0:
+                            time.sleep(s)
+                        continue
+
+                    raise RuntimeError('Unable to retrieve mapping')
+
+                date = data[0]["data"][0]["globalDate"]
+                date = dateutil.parser.parse(date)
+                dates.append(date)
 
         return dates
     except Exception:
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 85b3bd5..f1055c6 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.4
+  version: 0.0.5
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 27772a3..688de20 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.4",
+    version="0.0.5",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",
diff --git a/tests/test_download_data.py b/tests/test_download_data.py
index e5e30e1..c32f7c5 100644
--- a/tests/test_download_data.py
+++ b/tests/test_download_data.py
@@ -9,7 +9,7 @@ class TestDownloadData(TestCase):
         config = {
             'range': {
                 'startPulseId': 7314794868,
-                'endPulseId': 7314798868
+                'endPulseId': 7414798868
             },
 
             'parameters': {

From 7d18a1d38459df9f304aee504e03267bf355d634 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Tue, 11 Dec 2018 15:58:27 +0100
Subject: [PATCH 09/19] increase timeout increase logging

---
 cadump/cadump.py       | 5 ++++-
 conda-recipe/meta.yaml | 2 +-
 setup.py               | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 075590e..3cb49f0 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -92,6 +92,8 @@ def get_pulse_id_date_mapping(pulse_ids):
                      "fields": ["pulseId", "globalDate"]}
 
             for c in range(1):
+
+                logger.info("Retrieve mapping for pulse_id %d" % pulse_id)
                 # Query server
                 response = requests.post("https://data-api.psi.ch/sf/query", json=query)
 
@@ -102,6 +104,7 @@ def get_pulse_id_date_mapping(pulse_ids):
                 data = response.json()
 
                 if not pulse_id == data[0]["data"][0]["pulseId"]:
+                    logger.info("retrieval failed")
                     if c == 0:
                         ref_date = data[0]["data"][0]["globalDate"]
                         ref_date = dateutil.parser.parse(ref_date)
@@ -109,7 +112,7 @@ def get_pulse_id_date_mapping(pulse_ids):
                         now_date = datetime.datetime.now()
                         now_date = pytz.timezone('Europe/Zurich').localize(now_date)
 
-                        check_date = ref_date+datetime.timedelta(seconds=20)
+                        check_date = ref_date+datetime.timedelta(seconds=24)  # 20 seconds should be enough
                         delta_date = check_date - now_date
 
                         s = delta_date.seconds
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index f1055c6..b1ce859 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.5
+  version: 0.0.6
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 688de20..3b82973 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.5",
+    version="0.0.6",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",

From e12820231e341c70a5790ba0ae0b26a90436265c Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Thu, 13 Dec 2018 15:06:24 +0100
Subject: [PATCH 10/19] make a lot of retries ...

---
 cadump/cadump.py            | 3 ++-
 conda-recipe/meta.yaml      | 2 +-
 setup.py                    | 2 +-
 tests/test_download_data.py | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 3cb49f0..cf7d2ae 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -91,7 +91,7 @@ def get_pulse_id_date_mapping(pulse_ids):
                      "channels": ["SIN-CVME-TIFGUN-EVR0:BEAMOK"],
                      "fields": ["pulseId", "globalDate"]}
 
-            for c in range(1):
+            for c in range(10):
 
                 logger.info("Retrieve mapping for pulse_id %d" % pulse_id)
                 # Query server
@@ -126,6 +126,7 @@ def get_pulse_id_date_mapping(pulse_ids):
                 date = data[0]["data"][0]["globalDate"]
                 date = dateutil.parser.parse(date)
                 dates.append(date)
+                break
 
         return dates
     except Exception:
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index b1ce859..2cc8b87 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.6
+  version: 0.0.7
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 3b82973..5e4285b 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.6",
+    version="0.0.7",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",
diff --git a/tests/test_download_data.py b/tests/test_download_data.py
index c32f7c5..e5e30e1 100644
--- a/tests/test_download_data.py
+++ b/tests/test_download_data.py
@@ -9,7 +9,7 @@ class TestDownloadData(TestCase):
         config = {
             'range': {
                 'startPulseId': 7314794868,
-                'endPulseId': 7414798868
+                'endPulseId': 7314798868
             },
 
             'parameters': {

From 8230fcc1c1c9a657c4bc6a21d63b43cb9fe66d03 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 14 Dec 2018 10:27:19 +0100
Subject: [PATCH 11/19] add query expansion on the start

---
 cadump/cadump.py       | 32 ++++++++++++++++++++++++++++----
 conda-recipe/meta.yaml |  2 +-
 setup.py               |  2 +-
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index cf7d2ae..6f0de0e 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -2,6 +2,7 @@ from bottle import route, run, request, abort
 import json
 
 import data_api
+import data_api.client
 import requests
 import dateutil.parser
 import pytz
@@ -59,9 +60,16 @@ def download_data(config):
     new_filename = filename[:-3]+"_CA"+filename[-3:]
 
     logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date))
-    data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
-    logger.info("Persist data to hdf5 file")
-    data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
+    # data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
+    data = get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
+
+    if len(data) < 1:
+        logger.error("No data retrieved")
+        open(new_filename+"_NO_DATA", 'a').close()
+
+    else:
+        logger.info("Persist data to hdf5 file")
+        data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
 
 
 def read_channels(filename):
@@ -72,11 +80,27 @@ def read_channels(filename):
     for line in lines:
         line = line.strip()
         if line:  # if not empty line
-            channels.append(line) # remove all leading and trailing spaces
+            channels.append(line)  # remove all leading and trailing spaces
 
     return channels
 
 
+def get_data(channel_list, start=None, end=None, base_url=None):
+    query = {"range": {"startDate": datetime.datetime.isoformat(start), "endDate": datetime.datetime.isoformat(end), "startExpansion": True},
+             "channels": channel_list,
+             "fields": ["pulseId", "globalSeconds", "globalDate", "value", "eventCount"]}
+
+    response = requests.post(base_url + '/query', json=query)
+
+    # Check for successful return of data
+    if response.status_code != 200:
+        raise RuntimeError("Unable to retrieve data from server: ", response)
+
+    data = response.json()
+
+    return data_api.client._build_pandas_data_frame(data, index_field="globalDate")
+
+
 def get_pulse_id_date_mapping(pulse_ids):
 
     # See https://jira.psi.ch/browse/ATEST-897 for more details ...
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 2cc8b87..9f25753 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.7
+  version: 0.0.8
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 5e4285b..9072d51 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.7",
+    version="0.0.8",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",

From 6a4bb7c1782d9dbb43bd263cdb51c5d4d681b46a Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Tue, 29 Jan 2019 11:46:29 +0100
Subject: [PATCH 12/19] hacked comment parsing

---
 .gitignore                  | 1 +
 cadump/cadump.py            | 2 ++
 tests/channels.txt          | 3 ++-
 tests/test_download_data.py | 4 +---
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 77d9edc..3ed1302 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+build
 __pycache__
 .idea
 .pytest_cache
diff --git a/cadump/cadump.py b/cadump/cadump.py
index 6f0de0e..6d54b27 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -8,6 +8,7 @@ import dateutil.parser
 import pytz
 import datetime
 import time
+import re
 
 import logging
 logger = logging.getLogger("logger")
@@ -78,6 +79,7 @@ def read_channels(filename):
 
     channels = []
     for line in lines:
+        line = re.sub(r'\w*#.*', "", line)
         line = line.strip()
         if line:  # if not empty line
             channels.append(line)  # remove all leading and trailing spaces
diff --git a/tests/channels.txt b/tests/channels.txt
index 7033f9e..63868d2 100644
--- a/tests/channels.txt
+++ b/tests/channels.txt
@@ -2,4 +2,5 @@ S10-CPCL-VM1MGC:LOAD
 
 ONE
  TWO
-THREE
+# Comment that need to be removed
+THREE # one more comment at the end that need to be removed
diff --git a/tests/test_download_data.py b/tests/test_download_data.py
index e5e30e1..b85e1a0 100644
--- a/tests/test_download_data.py
+++ b/tests/test_download_data.py
@@ -27,9 +27,7 @@ class TestDownloadData(TestCase):
     def test_read_channels(self):
         channels = cadump.read_channels("channels.txt")
         logging.info(channels)
-        self.assertEqual(len(channels), 3)
-
-
+        self.assertEqual(len(channels), 4)
 
 
 if __name__ == '__main__':

From 1af5a3fecb00e21a466dc0382ecb33c40f302457 Mon Sep 17 00:00:00 2001
From: Dmitry Ozerov <Dmitry.Ozerov@psi.ch>
Date: Sat, 5 Oct 2019 15:07:06 +0200
Subject: [PATCH 13/19] protect against writting to /dev/null. try to retrieve
 data in case first attempt is not successful

---
 cadump/cadump.py            | 29 +++++++++++++++++++++++++----
 setup.py                    |  9 +++++++--
 tests/test_download_data.py |  4 ++--
 3 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 6d54b27..7dd5370 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -58,9 +58,12 @@ def download_data(config):
 
     # append _CA to the filename
     filename = config["parameters"]["output_file"]
-    new_filename = filename[:-3]+"_CA"+filename[-3:]
+    if filename != "/dev/null":
+        new_filename = filename[:-3]+"_CA"+filename[-3:]
+    else:
+        new_filename = None
 
-    logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date))
+    logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date) + " . From " + base_url)
     # data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
     data = get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
 
@@ -69,8 +72,9 @@ def download_data(config):
         open(new_filename+"_NO_DATA", 'a').close()
 
     else:
-        logger.info("Persist data to hdf5 file")
-        data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
+        if new_filename:
+            logger.info("Persist data to hdf5 file")
+            data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
 
 
 def read_channels(filename):
@@ -92,12 +96,29 @@ def get_data(channel_list, start=None, end=None, base_url=None):
              "channels": channel_list,
              "fields": ["pulseId", "globalSeconds", "globalDate", "value", "eventCount"]}
 
+    logger.info(query)
+
     response = requests.post(base_url + '/query', json=query)
 
     # Check for successful return of data
+    if response.status_code != 200:
+        #raise RuntimeError("Unable to retrieve data from server: ", response)
+        logger.info("Data retrievali failed, sleep for another time and try")
+
+        itry = 0
+        while itry < 5:
+            itry += 1
+            time.sleep(60)
+            response = requests.post(base_url + '/query', json=query)
+            if response.status_code == 200:
+                break
+            logger.info("Data retrieval failed, post attempt %d" % itry)
+
     if response.status_code != 200:
         raise RuntimeError("Unable to retrieve data from server: ", response)
 
+    logger.info("Data retieval is successful")
+
     data = response.json()
 
     return data_api.client._build_pandas_data_frame(data, index_field="globalDate")
diff --git a/setup.py b/setup.py
index 9072d51..69ac8de 100644
--- a/setup.py
+++ b/setup.py
@@ -6,5 +6,10 @@ setup(
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",
-    packages=["cadump"]
-)
\ No newline at end of file
+    packages=["cadump"],
+    entry_points={
+        'console_scripts': [
+            'cadump_server = cadump.cadump:main',
+        ],
+    }
+)
diff --git a/tests/test_download_data.py b/tests/test_download_data.py
index b85e1a0..87a7194 100644
--- a/tests/test_download_data.py
+++ b/tests/test_download_data.py
@@ -8,8 +8,8 @@ class TestDownloadData(TestCase):
     def test_download_data(self):
         config = {
             'range': {
-                'startPulseId': 7314794868,
-                'endPulseId': 7314798868
+                'startPulseId': 9618913001,
+                'endPulseId': 9618923000
             },
 
             'parameters': {

From 5ac7ce00e13d2cd047841ec767374912faf3f5da Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Thu, 30 Apr 2020 09:25:27 +0200
Subject: [PATCH 14/19] bump version

---
 conda-recipe/meta.yaml | 1 +
 setup.py               | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 9f25753..706effa 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -6,6 +6,7 @@ source:
     path: ..
 
 build:
+  noarch: python
   entry_points:
     - cadump_server = cadump.cadump:main
 
diff --git a/setup.py b/setup.py
index 69ac8de..48d48a1 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.8",
+    version="0.0.9",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",

From f78026eed5eec51484f68c6426e91f86fc075784 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 15 May 2020 15:44:18 +0200
Subject: [PATCH 15/19] fixed default mapping channel

---
 cadump/cadump.py       | 9 ++++++---
 conda-recipe/meta.yaml | 6 +++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 7dd5370..4419741 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -92,7 +92,9 @@ def read_channels(filename):
 
 
 def get_data(channel_list, start=None, end=None, base_url=None):
-    query = {"range": {"startDate": datetime.datetime.isoformat(start), "endDate": datetime.datetime.isoformat(end), "startExpansion": True},
+    query = {"range": {"startDate": datetime.datetime.isoformat(start),
+                       "endDate": datetime.datetime.isoformat(end),
+                       "startExpansion": True},
              "channels": channel_list,
              "fields": ["pulseId", "globalSeconds", "globalDate", "value", "eventCount"]}
 
@@ -132,10 +134,11 @@ def get_pulse_id_date_mapping(pulse_ids):
         dates = []
         for pulse_id in pulse_ids:
 
-            query = {"range": {"startPulseId": 0,"endPulseId": pulse_id},
+            query = {"range": {"startPulseId": 0,
+                               "endPulseId": pulse_id},
                      "limit": 1,
                      "ordering": "desc",
-                     "channels": ["SIN-CVME-TIFGUN-EVR0:BEAMOK"],
+                     "channels": ["SIN-CVME-TIFGUN-EVR0:BUNCH-1-OK"],
                      "fields": ["pulseId", "globalDate"]}
 
             for c in range(10):
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 706effa..e239f33 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.8
+  version: 0.0.9
 
 source:
     path: ..
@@ -13,7 +13,7 @@ build:
 requirements:
   build:
     - python
-    - data_api
+    - data_api >=0.7.6
   run:
     - python
-    - data_api
\ No newline at end of file
+    - data_api >=0.7.6
\ No newline at end of file

From 57730db1814513cb27b41f3d42732838af245274 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Mon, 18 May 2020 08:15:39 +0200
Subject: [PATCH 16/19] use channels list from request

---
 api.http                |  23 ++++++
 ca_default_channel_list | 179 ++++++++++++++++++++++++++++++++++++++++
 cadump/cadump.py        |   9 +-
 conda-recipe/meta.yaml  |   2 +-
 4 files changed, 211 insertions(+), 2 deletions(-)
 create mode 100644 api.http
 create mode 100644 ca_default_channel_list

diff --git a/api.http b/api.http
new file mode 100644
index 0000000..0fd0d2e
--- /dev/null
+++ b/api.http
@@ -0,0 +1,23 @@
+# For a quick start check out our HTTP Requests collection (Tools|HTTP Client|Open HTTP Requests Collection) or
+# paste cURL into the file and request will be converted to HTTP Request format.
+#
+# Following HTTP Request Live Templates are available:
+# * 'gtrp' and 'gtr' create a GET request with or without query parameters;
+# * 'ptr' and 'ptrp' create a POST request with a simple or parameter-like body;
+# * 'mptr' and 'fptr' create a POST request to submit a form with a text or file field (multipart/form-data);
+
+# curl -XPUT -d '{"range":{"startPulseId": 11281433214, "endPulseId": 7281489688}, "parameters":{"output_file":"test.h5"}}' http://localhost:10200/notify
+PUT http://localhost:10200/notify
+Content-Type: application/json
+
+{"range":{"startPulseId": 11876043548, "endPulseId": 11876044548}, "parameters":{"output_file":"test.h5"}}
+
+###
+
+PUT http://localhost:10200/notify
+Content-Type: application/json
+
+{"range":{"startPulseId": 11876043548, "endPulseId": 11876044548}, "parameters":{"output_file":"test.h5"}, "channels":  ["ABC"]}
+
+###
+
diff --git a/ca_default_channel_list b/ca_default_channel_list
new file mode 100644
index 0000000..cd23afe
--- /dev/null
+++ b/ca_default_channel_list
@@ -0,0 +1,179 @@
+SARES11-VPIG124-360:PRESSURE
+SARES11-VMFR124-350:PRESSURE
+
+SARES11-VMFR125-600:PRESSURE
+SARES11-VMCP125-610:PRESSURE
+SARES11-VMFR125-400:PRESSURE
+SARES11-VMCP125-410:PRESSURE
+SARES11-VMCP125-510:PRESSURE
+SARES11-EVSP-SAMPLE:SET_PRES
+SARES11-EVSP-010:SAMPLE
+SARES11-EVSP-010:INTERMEDIATE
+SARES11-EVSP-010:DIFFERENT
+
+
+SARES11-EVGG-A010:PRESSURE
+SARES11-EVGG-B010:PRESSURE
+SARES11-VMDI125-400:PRESSURE
+SARES11-VMDI125-500:PRESSURE
+SARES11-EVGG-C010:PRESSURE
+SARES11-EVGV-C010:PLC_OPEN
+SARES11-EVGV-D010:PLC_OPEN
+SARES11-EVGV-E010:PLC_OPEN
+SARES11-EVGV-E020:PLC_OPEN
+SARES11-EVGV-F010:PLC_OPEN
+
+SLAAR11-LMOT-M421:MOT.RBV
+SLAAR11-LMOT-M422:MOT.RBV
+SLAAR11-LMOT-M423:MOT.RBV
+
+SLAAR11-LMOT-M431:MOT.RBV
+SLAAR11-LMOT-M432:MOT.RBV
+SLAAR11-LMOT-M441:MOT.RBV
+SLAAR11-LMOT-M442:MOT.RBV
+SLAAR11-LMOT-M444:MOT.RBV
+SLAAR11-LENG-R452:VAL_GET
+SLAAR11-LMOT-M451:MOTOR_1.RBV
+
+SLAAR11-LDIO-LAS6891:SET_BO02
+SLAAR11-LPSYS-ESA:TRANS_OPEN
+
+SAROP11-PALMK118:RAVE
+SAROP11-PALMK118:LIVE
+SLAAR11-L-LAMCALC2:MOTOR_POS_SMAR_SC
+SLAAR11-LENG-R452:VAL_GET
+SARES11-XFOC125:LIN1:MOTRBV
+SARES11-XICM125:ROX1:MOTRBV
+SARES11-XICM125:TRX1:MOTRBV
+SARES11-XICM125:ROY1:MOTRBV
+
+SARES11-XMI125-C4P1:EXPOSURE
+SARES11-XMI125:FOCUS.RBV
+SARES11-XMI125:ZOOM.RBV
+SARES11-XILL125:LIGHTLEVEL_SP
+SARES11-XMI125:ROY1:MOTRBV
+SARES11-XMI125:ROZ1:MOTRBV
+
+SARES11-XOTA125:W_X.RBV
+SARES11-XOTA125:W_Y.RBV
+SARES11-XOTA125:W_Z.RBV
+SARES11-XOTA125:W_RX.RBV
+SARES11-XOTA125:W_RY.RBV
+SARES11-XOTA125:W_RZ.RBV
+SARES11-XOTA125:RAIL.RBV
+SARES11-XOTA125:MOTOR_Y1.RBV
+SARES11-XOTA125:MOTOR_Y2.RBV
+SARES11-XOTA125:MOTOR_Y3.RBV
+SARES11-XOTA125:MOTOR_X1.RBV
+SARES11-XOTA125:MOTOR_Z1.RBV
+SARES11-XOTA125:MOTOR_Z2.RBV
+SARES11-XOTA125:MODE_SP
+
+SARES11-XIZO125:TRX1:MOTRBV
+SARES11-XIZO125:TRX2:MOTRBV
+SARES11-XIZO125:TRY1:MOTRBV
+SARES11-XIZO125:TRY2:MOTRBV
+
+SARES10-XHPLC125:01:FLOW
+SARES10-XHPLC125:01:PRESSURE
+
+SARES10-XHPLC125:02:FLOW
+SARES10-XHPLC125:02:PRESSURE
+
+SARES11-XSAM125:MOTOR_X1.RBV
+SARES11-XSAM125:MOTOR_Y1.RBV
+SARES11-XSAM125:MOTOR_Z1.RBV
+
+SARES11-XCRY125:CRY_1.RBV
+SARES11-XCRY125:CRY_2.RBV
+SARES11-XCRY125:TRZ11:MOTRBV
+SARES11-XCRY125:ROY11:MOTRBV
+SARES11-XCRY125:ROX11:MOTRBV
+SARES11-XCRY125:TRZ21:MOTRBV
+SARES11-XCRY125:ROY21:MOTRBV
+SARES11-XCRY125:ROX21:MOTRBV
+SARES11-XCRY125:TRZ12:MOTRBV
+SARES11-XCRY125:ROY12:MOTRBV
+SARES11-XCRY125:ROX12:MOTRBV
+SARES11-XCRY125:TRZ22:MOTRBV
+SARES11-XCRY125:ROY22:MOTRBV
+SARES11-XCRY125:ROX22:MOTRBV
+
+SARBD01-DBPM040:Q1
+SAROP11-ARAMIS:ENERGY
+SAROP11-ARAMIS:MODE
+SARFE10-PBPG050:PHOTON-ENERGY-PER-PULSE-AVG
+SGE-OP2E-ARAMIS:ESA_PH2E_Y1
+SGE-OP2E-ARAMIS:ESA_PH2E_Y2
+SGE-OP2E-ARAMIS:ESA_PH2E_X1
+SGE-OP2E-ARAMIS:ESA_PH2E_X2
+SARCL02-DCOL290:ENE
+SARBD01-DBPM040:Q1
+SARBD01-MBND100:P-READ
+SARUN03-UIND030:K_SET
+SARUN04-UIND030:K_SET
+SARUN05-UIND030:K_SET
+SARUN06-UIND030:K_SET
+SARUN07-UIND030:K_SET
+SARUN08-UIND030:K_SET
+SARUN09-UIND030:K_SET
+SARUN10-UIND030:K_SET
+SARUN11-UIND030:K_SET
+SARUN12-UIND030:K_SET
+SARUN13-UIND030:K_SET
+SARUN14-UIND030:K_SET
+
+SIN-TIMAST-TMA:Beam-Appl-Freq-RB
+SIN-TIMAST-TMA:Evt-20-Delay-RB
+SIN-TIMAST-TMA:Evt-20-Freq-I
+SIN-TIMAST-TMA:Evt-50-Delay-RB
+SIN-TIMAST-TMA:Evt-50-Freq-I
+SIN-TIMAST-TMA:Evt-29-Delay-RB
+SIN-TIMAST-TMA:Evt-29-Freq-I
+SIN-TIMAST-TMA:Evt-24-Delay-RB
+SIN-TIMAST-TMA:Evt-24-Freq-I
+
+SARFE10-OATT053:TRANS_SP
+SARFE10-OATT053:TRANS_RB
+SARFE10-OATT053:TRANS3EDHARM_RB
+SARFE10-OATT053:ENERGY
+SAROP11-OATT120:TRANS_SP
+SAROP11-OATT120:TRANS_RB
+SAROP11-OATT120:TRANS3EDHARM_RB
+SAROP11-OATT120:ENERGY
+
+SAROP11-OPPI110:MOTOR_X1.RBV
+SAROP11-OPPI110:MOTOR_Y1.RBV
+SAROP11-OPPI110:PRESET_SP
+SGE-CPCW-71-EVR0:FrontUnivOut15-Ena-SP
+SGE-CPCW-71-EVR0:FrontUnivOut15-Src-SP
+
+SARFE10-OAPU044:MOTOR_X
+SARFE10-OAPU044:MOTOR_Y
+SARFE10-OAPU044:MOTOR_W
+SARFE10-OAPU044:MOTOR_H
+SARFE10-OAPU044:MOTOR_X.RBV
+SARFE10-OAPU044:MOTOR_Y.RBV
+SARFE10-OAPU044:MOTOR_W.RBV
+SARFE10-OAPU044:MOTOR_H.RBV
+SARFE10-OAPU044:MOTOR_AX1.RBV
+SARFE10-OAPU044:MOTOR_AX2.RBV
+SARFE10-OAPU044:MOTOR_AY1.RBV
+SARFE10-OAPU044:MOTOR_AY2.RBV
+SARFE10-OAPU044:MOTOR_BX1.RBV
+SARFE10-OAPU044:MOTOR_BX2.RBV
+SARFE10-OAPU044:MOTOR_BY1.RBV
+SARFE10-OAPU044:MOTOR_BY2.RBV
+
+SAROP11-OAPU104:MOTOR_X
+SAROP11-OAPU104:MOTOR_Y
+SAROP11-OAPU104:MOTOR_W
+SAROP11-OAPU104:MOTOR_H
+
+SAROP11-OAPU120:MOTOR_X
+SAROP11-OAPU120:MOTOR_Y
+SAROP11-OAPU120:MOTOR_W
+SAROP11-OAPU120:MOTOR_H
+
+SLAAR01-TLSY-EPL:JITTERMON
+
diff --git a/cadump/cadump.py b/cadump/cadump.py
index 4419741..b56731c 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -52,6 +52,13 @@ def download_data(config):
     start_pulse = config["range"]["startPulseId"]
     end_pulse = config["range"]["endPulseId"]
 
+    # Overwrite the channels list if specified in the request
+    channels = channel_list
+    if "channels" in config:
+        channels = config["channels"]
+
+    logger.info("Retrieve data for channels: ", channels)
+
     logger.info("Retrieve pulse-id / data mapping for pulse ids")
     # start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
     start_date, end_date = get_pulse_id_date_mapping([start_pulse, end_pulse])
@@ -65,7 +72,7 @@ def download_data(config):
 
     logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date) + " . From " + base_url)
     # data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
-    data = get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
+    data = get_data(channels, start=start_date, end=end_date, base_url=base_url)
 
     if len(data) < 1:
         logger.error("No data retrieved")
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index e239f33..1dcbdb0 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.9
+  version: 0.0.10
 
 source:
     path: ..

From 87b684d06b7c46f76ed3e314ed0006dc57489330 Mon Sep 17 00:00:00 2001
From: Dmitry Ozerov <dmitry.ozerov@psi.ch>
Date: Mon, 18 May 2020 10:37:26 +0200
Subject: [PATCH 17/19] take url from request, if provided. Don't append _CA to
 provided in request filename. Better error handling when data_api retrieve
 doesn't work

---
 cadump/cadump.py       | 20 ++++++++++++--------
 conda-recipe/meta.yaml |  4 ++--
 setup.py               |  2 +-
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index b56731c..51a7a2b 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -47,7 +47,6 @@ def put_document():
 def download_data(config):
 
     logger.info("Dump data to hdf5 ...")
-    # logger.info(config)
 
     start_pulse = config["range"]["startPulseId"]
     end_pulse = config["range"]["endPulseId"]
@@ -57,22 +56,24 @@ def download_data(config):
     if "channels" in config:
         channels = config["channels"]
 
-    logger.info("Retrieve data for channels: ", channels)
+    logger.info("Retrieve data for channels: %s" % channels)
+
+    if "retrieval_url" in config:
+        new_base_url = config["retrieval_url"]
+    else:
+        new_base_url = base_url
 
     logger.info("Retrieve pulse-id / data mapping for pulse ids")
-    # start_date, end_date = data_api.get_global_date([start_pulse, end_pulse])
     start_date, end_date = get_pulse_id_date_mapping([start_pulse, end_pulse])
 
-    # append _CA to the filename
     filename = config["parameters"]["output_file"]
     if filename != "/dev/null":
-        new_filename = filename[:-3]+"_CA"+filename[-3:]
+        new_filename = filename
     else:
         new_filename = None
 
-    logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date) + " . From " + base_url)
-    # data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
-    data = get_data(channels, start=start_date, end=end_date, base_url=base_url)
+    logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date) + " . From " + new_base_url)
+    data = get_data(channels, start=start_date, end=end_date, base_url=new_base_url)
 
     if len(data) < 1:
         logger.error("No data retrieved")
@@ -160,6 +161,9 @@ def get_pulse_id_date_mapping(pulse_ids):
 
                 data = response.json()
 
+                if len(data[0]["data"]) == 0 or not "pulseID" in data[0]["data"][0]:
+                    raise RuntimeError("Didn't get good responce from data_api : %s " % data)
+
                 if not pulse_id == data[0]["data"][0]["pulseId"]:
                     logger.info("retrieval failed")
                     if c == 0:
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 1dcbdb0..b6675a1 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.10
+  version: 0.0.11
 
 source:
     path: ..
@@ -16,4 +16,4 @@ requirements:
     - data_api >=0.7.6
   run:
     - python
-    - data_api >=0.7.6
\ No newline at end of file
+    - data_api >=0.7.6
diff --git a/setup.py b/setup.py
index 48d48a1..1cd3acc 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.9",
+    version="0.0.11",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",

From 22381040408e47e4489ba9c1bb1f2d10e867784d Mon Sep 17 00:00:00 2001
From: Dmitry Ozerov <dmitry.ozerov@psi.ch>
Date: Mon, 18 May 2020 11:23:22 +0200
Subject: [PATCH 18/19] mistype

---
 cadump/cadump.py       | 3 +--
 conda-recipe/meta.yaml | 2 +-
 setup.py               | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index 51a7a2b..612aa39 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -112,7 +112,6 @@ def get_data(channel_list, start=None, end=None, base_url=None):
 
     # Check for successful return of data
     if response.status_code != 200:
-        #raise RuntimeError("Unable to retrieve data from server: ", response)
         logger.info("Data retrievali failed, sleep for another time and try")
 
         itry = 0
@@ -161,7 +160,7 @@ def get_pulse_id_date_mapping(pulse_ids):
 
                 data = response.json()
 
-                if len(data[0]["data"]) == 0 or not "pulseID" in data[0]["data"][0]:
+                if len(data[0]["data"]) == 0 or not "pulseId" in data[0]["data"][0]:
                     raise RuntimeError("Didn't get good responce from data_api : %s " % data)
 
                 if not pulse_id == data[0]["data"][0]["pulseId"]:
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index b6675a1..e46d052 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.11
+  version: 0.0.12
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 1cd3acc..d4358df 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.11",
+    version="0.0.12",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",

From 89228a57f2936f8cf2e6fb9c3a37c337c5b4d011 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Mon, 18 May 2020 11:38:09 +0200
Subject: [PATCH 19/19] update readme

---
 Readme.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Readme.md b/Readme.md
index 4d96c6f..5f6ab51 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,7 +1,12 @@
 # Overview
+
 Simple server to dump Epics Channel Access data to an HDF5 file.
 The server gets an http callback from the Broker whenever there was an acquisition.
 
+
+__Note: THIS IS/WAS A FRIDAY AFTERNOON HACK TO MAKE THE SWISSFEL DAQ WORK__
+
+
 The format of the request is as follows:
 ```
 {
@@ -26,4 +31,4 @@ Right now this server needs to run on the same server than the
 
 ```bash
 curl -XPUT -d '{"range":{"startPulseId": 7281433214, "endPulseId": 7281489688}, "parameters":{"output_file":"test.h5"}}' http://localhost:10200/notify
-``` 
\ No newline at end of file
+```