From 8230fcc1c1c9a657c4bc6a21d63b43cb9fe66d03 Mon Sep 17 00:00:00 2001
From: Simon Ebner <simon.ebner@psi.ch>
Date: Fri, 14 Dec 2018 10:27:19 +0100
Subject: [PATCH] add query expansion on the start

---
 cadump/cadump.py       | 32 ++++++++++++++++++++++++++++----
 conda-recipe/meta.yaml |  2 +-
 setup.py               |  2 +-
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/cadump/cadump.py b/cadump/cadump.py
index cf7d2ae..6f0de0e 100644
--- a/cadump/cadump.py
+++ b/cadump/cadump.py
@@ -2,6 +2,7 @@ from bottle import route, run, request, abort
 import json
 
 import data_api
+import data_api.client
 import requests
 import dateutil.parser
 import pytz
@@ -59,9 +60,16 @@ def download_data(config):
     new_filename = filename[:-3]+"_CA"+filename[-3:]
 
     logger.info("Retrieving data for interval start: " + str(start_date) + " end: " + str(end_date))
-    data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
-    logger.info("Persist data to hdf5 file")
-    data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
+    # data = data_api.get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
+    data = get_data(channel_list, start=start_date, end=end_date, base_url=base_url)
+
+    if len(data) < 1:
+        logger.error("No data retrieved")
+        open(new_filename+"_NO_DATA", 'a').close()
+
+    else:
+        logger.info("Persist data to hdf5 file")
+        data_api.to_hdf5(data, new_filename, overwrite=True, compression=None, shuffle=False)
 
 
 def read_channels(filename):
@@ -72,11 +80,27 @@ def read_channels(filename):
     for line in lines:
         line = line.strip()
         if line:  # if not empty line
-            channels.append(line) # remove all leading and trailing spaces
+            channels.append(line)  # remove all leading and trailing spaces
 
     return channels
 
 
+def get_data(channel_list, start=None, end=None, base_url=None):
+    query = {"range": {"startDate": datetime.datetime.isoformat(start), "endDate": datetime.datetime.isoformat(end), "startExpansion": True},
+             "channels": channel_list,
+             "fields": ["pulseId", "globalSeconds", "globalDate", "value", "eventCount"]}
+
+    response = requests.post(base_url + '/query', json=query)
+
+    # Check for successful return of data
+    if response.status_code != 200:
+        raise RuntimeError("Unable to retrieve data from server: ", response)
+
+    data = response.json()
+
+    return data_api.client._build_pandas_data_frame(data, index_field="globalDate")
+
+
 def get_pulse_id_date_mapping(pulse_ids):
 
     # See https://jira.psi.ch/browse/ATEST-897 for more details ...
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
index 2cc8b87..9f25753 100644
--- a/conda-recipe/meta.yaml
+++ b/conda-recipe/meta.yaml
@@ -1,6 +1,6 @@
 package:
   name: cadump
-  version: 0.0.7
+  version: 0.0.8
 
 source:
     path: ..
diff --git a/setup.py b/setup.py
index 5e4285b..9072d51 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="cadump",
-    version="0.0.7",
+    version="0.0.8",
     author="Paul Scherrer Institute",
     author_email="daq@psi.ch",
     description="Interface to dump data from archiver/databuffer",