diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..73358ad --- /dev/null +++ b/.gitignore @@ -0,0 +1,139 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + diff --git a/example_scicat.py b/example_scicat.py new file mode 100755 index 0000000..8e8d203 --- /dev/null +++ b/example_scicat.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +from scilog import SciCat + +url = "https://dacat.psi.ch/api/v3/" +cat = SciCat(url) + +props = cat.proposals +nprops = len(props) +print(f"got {nprops} proposals") +print(props[0]) + + + diff --git a/example_scilog.py b/example_scilog.py new file mode 100755 index 0000000..6a48e2b --- /dev/null +++ b/example_scilog.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("pgroup", help="Expected form: p12345") + +clargs = parser.parse_args() +pgroup = clargs.pgroup + + +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +from scilog import SciLog + +url = "https://lnode2.psi.ch/api/v1" +log = SciLog(url) +#print(log.token) +loc = log.get_snippets(title="location", ownerGroup="admin") + +assert len(loc) == 1 +loc_id = loc[0]["id"] +print(loc_id) + +lb = log.get_snippets(snippetType="logbook", ownerGroup=pgroup) + +assert len(lb) == 1 +lb_id = lb[0]["id"] +print(lb_id) + +res = log.post_snippet(snippetType="paragraph", ownerGroup=pgroup, parentId=lb_id, textcontent="

from python

") +print(res) + +snips = log.get_snippets(snippetType="paragraph", ownerGroup=pgroup) +print(snips) + + + diff --git a/scilog/__init__.py b/scilog/__init__.py new file mode 100644 index 0000000..0fd2f3d --- /dev/null +++ b/scilog/__init__.py @@ -0,0 +1,5 @@ + +from .scicat import SciCat +from .scilog import SciLog + + diff --git a/scilog/autherror.py b/scilog/autherror.py new file mode 100644 index 0000000..7268efe --- /dev/null +++ b/scilog/autherror.py @@ -0,0 +1,5 @@ + +class AuthError(Exception): + pass + + diff --git a/scilog/config.py b/scilog/config.py new file mode 100644 index 0000000..bcb69a8 --- /dev/null +++ b/scilog/config.py @@ -0,0 +1,44 @@ +from pathlib import Path +import json + + +class Config(dict): + + def __init__(self, fname, folder=None): + if folder is not None: + folder = Path(folder) + else: + folder = Path.home() + self.fname = folder / fname + content = self._load() + super().update(content) + + def __setitem__(self, name, value): + self.update(**{name: value}) + + def update(self, **kwargs): + super().update(**kwargs) + self._save() + + def _load(self): + fn = self.fname + if fn.exists(): + return json_load(fn) + else: + return {} + + def _save(self): + json_save(self, self.fname) + + + +def json_save(what, filename, *args, indent=4, sort_keys=True, **kwargs): + with open(filename, "w") as f: + json.dump(what, f, *args, indent=indent, sort_keys=sort_keys, **kwargs) + +def json_load(filename, *args, **kwargs): + with open(filename, "r") as f: + return json.load(f, *args, **kwargs) + + + diff --git a/scilog/mkfilt.py b/scilog/mkfilt.py new file mode 100644 index 0000000..d11097b --- /dev/null +++ b/scilog/mkfilt.py @@ -0,0 +1,11 @@ +import json + + +def make_filter(**kwargs): + items = [{k: v} for k, v in kwargs.items()] + filt = {"where": {"and": items}} + filt = json.dumps(filt) + return {"filter": filt} + + + diff --git a/scilog/scicat.py b/scilog/scicat.py new file mode 100644 index 0000000..59ca80a --- /dev/null +++ b/scilog/scicat.py @@ -0,0 +1,69 @@ +import getpass +from .config import Config +from .utils import post_request, get_request +from .autherror import AuthError + + +AUTH_HEADERS = { + "Content-type": "application/json", + "Accept": "application/json" +} + + +class SciCat: + + def __init__(self, address): + self.address = address.rstrip("/") + self._token = None + self.config = Config(".scicat-tokens") + + def __repr__(self): + return f"SciCat @ {self.address}" + + @property + def proposals(self): + url = self.address + "/proposals" + headers = self.auth_headers + return get_request(url, headers=headers) + + + @property + def auth_headers(self): + headers = AUTH_HEADERS.copy() + headers["Authorization"] = self.token + return headers + + @property + def token(self): + username = getpass.getuser() + password = getpass.getpass(prompt=f"SciCat password for {username}: ") + token = self._token + if token is None: + try: + token = self.config[username] + except KeyError: + token = self.authenticate(username, password) + self.config[username] = self._token = token + return token + + def authenticate(self, username, password): + url = self.address + "/users/login" + auth_payload = { + "username": username, + "password": password + } + res = post_request(url, auth_payload, AUTH_HEADERS) + try: + token = res["id"] + except KeyError as e: + raise SciCatAuthError(res) from e + else: + return token + + + +class SciCatAuthError(AuthError): + pass + + + diff --git a/scilog/scilog.py b/scilog/scilog.py new file mode 100644 index 0000000..3f20c96 --- /dev/null +++ b/scilog/scilog.py @@ -0,0 +1,77 @@ +import getpass +from .config import Config +from .utils import post_request, get_request +from .autherror import AuthError +from .mkfilt import make_filter + + +AUTH_HEADERS = { + "Content-type": "application/json", + "Accept": "application/json" +} + + +class SciLog: + + def __init__(self, address): + self.address = address.rstrip("/") + self._token = None + self.config = Config(".scilog-tokens") + + def __repr__(self): + return f"SciLog @ {self.address}" + + + def get_snippets(self, **kwargs): + url = self.address + "/basesnippets" + params = make_filter(**kwargs) + headers = self.auth_headers + return get_request(url, params=params, headers=headers) + + def post_snippet(self, **kwargs): + url = self.address + "/basesnippets" + payload = kwargs + headers = self.auth_headers + return post_request(url, payload=payload, headers=headers) + + + @property + def auth_headers(self): + headers = AUTH_HEADERS.copy() + headers["Authorization"] = self.token + return headers + + @property + def token(self): + username = getpass.getuser() + token = self._token + if token is None: + try: + token = self.config[username] + except KeyError: + password = getpass.getpass(prompt=f"SciLog password for {username}: ") + token = self.authenticate(username, password) + self.config[username] = self._token = token + return token + + def authenticate(self, username, password): + url = self.address + "/users/login" + auth_payload = { + "principal": username, + "password": password + } + res = post_request(url, auth_payload, AUTH_HEADERS) + try: + token = "Bearer " + res["token"] + except KeyError as e: + raise SciLogAuthError(res) from e + else: + return token + + + +class SciLogAuthError(AuthError): + pass + + + diff --git a/scilog/utils.py b/scilog/utils.py new file mode 100644 index 0000000..b7d0f69 --- /dev/null +++ b/scilog/utils.py @@ -0,0 +1,20 @@ +import requests + + +#TODO: add params/payload and response validation + + +def get_request(url, params=None, headers=None, timeout=10): + response = requests.get(url, params=params, headers=headers, timeout=timeout, verify=False).json() + return response + +def post_request(url, payload=None, headers=None, timeout=10): + response = requests.post(url, json=payload, headers=headers, timeout=timeout, verify=False).json() + return response + + +def typename(obj): + return type(obj).__name__ + + + diff --git a/update_locations.py b/update_locations.py new file mode 100755 index 0000000..d1c22a6 --- /dev/null +++ b/update_locations.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +from scicat import SciCat, SciLog + + +def prepare_location_snippet(log): + snips = log.get_snippets(title="location", ownerGroup="admin") + + if snips: + assert len(snips) == 1 + loc_id = snips[0]["id"] + return loc_id + + new_loc = { + "ownerGroup": "admin", + "accessGroups": ["customer"], + "isPrivate": True, + "title": "location", + "snippetType": "paragraph", + } + + snip = log.post_snippet(**new_loc) + loc_id = snip["id"] + return loc_id + + +def update_locations_and_proposals(log, loc_id, proposals): + _accessGroups, locations, proposalsStorage = _collect_data(proposals) + locationStorage = _update_locations(log, loc_id, locations) + _update_proposals(log, locationStorage, proposalsStorage) + + +def _collect_data(proposals): + accessGroups = set() + locations = set() + proposalsStorage = [] + + for prop in proposals: + for ag in prop["accessGroups"]: + accessGroups.add(ag) + + loc = prop["MeasurementPeriodList"][0]["instrument"] + locations.add(loc) + + proposalsStorage.append({ + "ownerGroup": prop["ownerGroup"], + "abstract": prop["abstract"], + "title": prop["title"], + "location": prop["MeasurementPeriodList"][0]["instrument"] + }) + + return accessGroups, locations, proposalsStorage + + +def _update_locations(log, loc_id, locations): + locationStorage = dict() + + for loc in locations: + #TODO: move this to the first loop? + if loc[:4] != "/PSI": + raise RuntimeError("Unexpected facility prefix") + + snips = log.get_snippets(parentId=loc_id, location=loc) + + if snips: + assert len(snips) == 1 + snip = snips[0] + locationStorage[loc] = snip + continue + + group = loc[5:].replace("/", "").lower() + new_snip = { + "ownerGroup": group, + "accessGroups": ["customer"], + "isPrivate": True, + "title": loc.split("/")[-1], + "location": loc, + "contact": group + "@psi.ch", + "snippetType": "image", + "parentId": loc_id, + "file": "files/default_logbook_icon.jpg" + } + + if "thumbnail" in loc: + new_snip["file"] = loc["thumbnail"] + + snip = log.post_request(**new_snip) + locationStorage[loc] = snip + + return locationStorage + + +def _update_proposals(log, locationStorage, proposalsStorage): + for proposal in proposalsStorage: + ownerGroup = proposal["ownerGroup"] + + loc = proposal["location"] + loc = locationStorage[loc] + + new_snip = { + "ownerGroup": ownerGroup, + "accessGroups": [loc["ownerGroup"]], + "isPrivate": False, + "name": proposal["title"], + "location": loc["id"], + "description": proposal["abstract"], + "snippetType": "logbook" + } + + if "file" in loc: + new_snip["thumbnail"] = loc["file"] + if not new_snip["name"]: + new_snip["name"] = ownerGroup + if not new_snip["description"]: + new_snip["description"] = "No proposal found." + + snips = log.get_snippets(snippetType="logbook", ownerGroup=ownerGroup) + + if snips: + print(f"Logbook exists already for ownerGroup {ownerGroup}") + continue + + print(f"Adding new logbook for ownerGroup {ownerGroup}") + snip = log.post_snippet(**new_snip) + print(snip) + + + + + +if __name__ == "__main__": + url = "https://dacat.psi.ch/api/v3/" + cat = SciCat(url) + props = cat.proposals + + url = "https://lnode2.psi.ch/api/v1" + log = SciLog(url) + loc_id = prepare_location_snippet(log) + + update_locations_and_proposals(log, loc_id, props) + + +