apps4ops/common
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added 3rd party packages, elog, bigtree

Browse Source
This commit is contained in:
chrin
2024-02-27 15:40:00 +01:00
parent 277c22f800
commit 6b59fe16ce
69 changed files with 17449 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

571
python310/packages/elog/logbook.py Normal file
View File

@@ -0,0 +1,571 @@
import requests
import urllib.parse
import os
import builtins
import re
from elog.logbook_exceptions import *
from datetime import datetime
class Logbook(object):
"""
Logbook provides methods to interface with logbook on location: "server:port/subdir/logbook". User can create,
edit, delete logbook messages.
"""
def __init__(self, hostname, logbook='', port=None, user=None, password=None, subdir='', use_ssl=True,
encrypt_pwd=True):
"""
:param hostname: elog server hostname. If whole url is specified here, it will be parsed and arguments:
"logbook, port, subdir, use_ssl" will be overwritten by parsed values.
:param logbook: name of the logbook on the elog server
:param port: elog server port (if not specified will default to '80' if use_ssl=False or '443' if use_ssl=True
:param user: username (if authentication needed)
:param password: password (if authentication needed) Password will be encrypted with sha256 unless
encrypt_pwd=False (default: True)
:param subdir: subdirectory of logbooks locations
:param use_ssl: connect using ssl (ignored if url starts with 'http://'' or 'https://'?
:param encrypt_pwd: To avoid exposing password in the code, this flag can be set to False and password
will then be handled as it is (user needs to provide sha256 encrypted password with
salt= '' and rounds=5000)
:return:
"""
hostname = hostname.strip()
# parse url to see if some parameters are defined with url
parsed_url = urllib.parse.urlsplit(hostname)
# ---- handle SSL -----
# hostname must be modified according to use_ssl flag. If hostname starts with https:// or http://
# the use_ssl flag is ignored
url_scheme = parsed_url.scheme
if url_scheme == 'http':
use_ssl = False
elif url_scheme == 'https':
use_ssl = True
elif not url_scheme:
# add http or https
if use_ssl:
url_scheme = 'https'
else:
url_scheme = 'http'
# ---- handle port -----
# 1) by default use port defined in the url
# 2) remove any 'default' ports such as 80 for http and 443 for https
# 3) if port not defined in url and not 'default' add it to netloc
netloc = parsed_url.netloc
if netloc == "" and "localhost" in hostname:
netloc = 'localhost'
netloc_split = netloc.split(':')
if len(netloc_split) > 1:
# port defined in url --> remove if needed
port = netloc_split[1]
if (port == 80 and not use_ssl) or (port == 443 and use_ssl):
netloc = netloc_split[0]
else:
# add port info if needed
if port is not None and not (port == 80 and not use_ssl) and not (port == 443 and use_ssl):
netloc += ':{}'.format(port)
# ---- handle subdir and logbook -----
# parsed_url.path = /<subdir>/<logbook>/
# Remove last '/' for easier parsing
url_path = parsed_url.path
if url_path.endswith('/'):
url_path = url_path[:-1]
splitted_path = url_path.split('/')
if url_path and len(splitted_path) > 1:
# If here ... then at least some part of path is defined.
# If logbook defined --> treat path current path as subdir and add logbook at the end
# to define the full path. Else treat existing path as <subdir>/<logbook>.
# Put first and last '/' back on its place
if logbook:
url_path += '/{}'.format(logbook)
else:
logbook = splitted_path[-1]
else:
# There is nothing. Use arguments.
url_path = subdir + '/' + logbook
# urllib.parse.quote replaces special characters with %xx escapes
# self._logbook_path = urllib.parse.quote('/' + url_path + '/').replace('//', '/')
self._logbook_path = ('/' + url_path + '/').replace('//', '/')
self._url = url_scheme + '://' + netloc + self._logbook_path
self.logbook = logbook
self._user = user
self._password = _handle_pswd(password, encrypt_pwd)
def post(self, message, msg_id=None, reply=False, attributes=None, attachments=None, encoding=None,
**kwargs):
"""
Posts message to the logbook. If msg_id is not specified new message will be created, otherwise existing
message will be edited, or a reply (if reply=True) to it will be created. This method returns the msg_id
of the newly created message.
:param message: string with message text
:param msg_id: ID number of message to edit or reply. If not specified new message is created.
:param reply: If 'True' reply to existing message is created instead of editing it
:param attributes: Dictionary of attributes. Following attributes are used internally by the elog and will be
ignored: Text, Date, Encoding, Reply to, In reply to, Locked by, Attachment
:param attachments: list of:
- file like objects which read() will return bytes (if file_like_object.name is not
defined, default name "attachment<i>" will be used.
- paths to the files
All items will be appended as attachment to the elog entry. In case of unknown
attachment an exception LogbookInvalidAttachment will be raised.
:param encoding: Defines encoding of the message. Can be: 'plain' -> plain text, 'html'->html-text,
'ELCode' --> elog formatting syntax
:param kwargs: Anything in the kwargs will be interpreted as attribute. e.g.: logbook.post('Test text',
Author='Rok Vintar), "Author" will be sent as an attribute. If named same as one of the
attributes defined in "attributes", kwargs will have priority.
:return: msg_id
"""
attributes = attributes or {}
attributes = {**attributes, **kwargs} # kwargs as attributes with higher priority
attachments = attachments or []
if encoding is not None:
if encoding not in ['plain', 'HTML', 'ELCode']:
raise LogbookMessageRejected('Invalid message encoding. Valid options: plain, HTML, ELCode.')
attributes['Encoding'] = encoding
attributes_to_edit = dict()
if msg_id:
# Message exists, we can continue
if reply:
# Verify that there is a message on the server, otherwise do not reply to it!
self._check_if_message_on_server(msg_id) # raises exception in case of none existing message
attributes['reply_to'] = str(msg_id)
else: # Edit existing
attributes['edit_id'] = str(msg_id)
attributes['skiplock'] = '1'
# Handle existing attachments
msg_to_edit, attributes_to_edit, attach_to_edit = self.read(msg_id)
i = 0
for attachment in attach_to_edit:
if attachment:
# Existing attachments must be passed as regular arguments attachment<i> with value= file name
# Read message returnes full urls to existing attachments:
# <hostname>:[<port>][/<subdir]/<logbook>/<msg_id>/<file_name>
attributes['attachment' + str(i)] = os.path.basename(attachment)
i += 1
for attribute, data in attributes.items():
new_data = attributes.get(attribute)
if new_data is not None:
attributes_to_edit[attribute] = new_data
else:
# As we create a new message, specify creation time if not already specified in attributes
if 'When' not in attributes:
attributes['When'] = int(datetime.now().timestamp())
if not attributes_to_edit:
attributes_to_edit = attributes
# Remove any attributes that should not be sent
_remove_reserved_attributes(attributes_to_edit)
if attachments:
files_to_attach, objects_to_close = self._prepare_attachments(attachments)
else:
objects_to_close = list()
files_to_attach = list()
# Make requests module think that Text is a "file". This is the only way to force requests to send data as
# multipart/form-data even if there are no attachments. Elog understands only multipart/form-data
files_to_attach.append(('Text', ('', message)))
# Base attributes are common to all messages
self._add_base_msg_attributes(attributes_to_edit)
# Keys in attributes cannot have certain characters like whitespaces or dashes for the http request
attributes_to_edit = _replace_special_characters_in_attribute_keys(attributes_to_edit)
try:
response = requests.post(self._url, data=attributes_to_edit, files=files_to_attach, allow_redirects=False,
verify=False)
# Validate response. Any problems will raise an Exception.
resp_message, resp_headers, resp_msg_id = _validate_response(response)
# Close file like objects that were opened by the elog (if path
for file_like_object in objects_to_close:
if hasattr(file_like_object, 'close'):
file_like_object.close()
except requests.RequestException as e:
# Check if message on server.
self._check_if_message_on_server(msg_id) # raises exceptions if no message or no response from server
# If here: message is on server but cannot be downloaded (should never happen)
raise LogbookServerProblem('Cannot access logbook server to post a message, ' + 'because of:\n' +
'{0}'.format(e))
# Any error before here should raise an exception, but check again for nay case.
if not resp_msg_id or resp_msg_id < 1:
raise LogbookInvalidMessageID('Invalid message ID: ' + str(resp_msg_id) + ' returned')
return resp_msg_id
def read(self, msg_id):
"""
Reads message from the logbook server and returns tuple of (message, attributes, attachments) where:
message: string with message body
attributes: dictionary of all attributes returned by the logbook
attachments: list of urls to attachments on the logbook server
:param msg_id: ID of the message to be read
:return: message, attributes, attachments
"""
request_headers = dict()
if self._user or self._password:
request_headers['Cookie'] = self._make_user_and_pswd_cookie()
try:
self._check_if_message_on_server(msg_id) # raises exceptions if no message or no response from server
response = requests.get(self._url + str(msg_id) + '?cmd=download', headers=request_headers,
allow_redirects=False, verify=False)
# Validate response. If problems Exception will be thrown.
resp_message, resp_headers, resp_msg_id = _validate_response(response)
except requests.RequestException as e:
# If here: message is on server but cannot be downloaded (should never happen)
raise LogbookServerProblem('Cannot access logbook server to read the message with ID: ' + str(msg_id) +
'because of:\n' + '{0}'.format(e))
# Parse message to separate message body, attributes and attachments
attributes = dict()
attachments = list()
returned_msg = resp_message.decode('utf-8', 'ignore').splitlines()
delimiter_idx = returned_msg.index('========================================')
message = '\n'.join(returned_msg[delimiter_idx + 1:])
for line in returned_msg[0:delimiter_idx]:
line = line.split(': ')
data = ''.join(line[1:])
if line[0] == 'Attachment':
attachments = data.split(',')
# Here are only attachment names, make a full url out of it, so they could be
# recognisable by others, and downloaded if needed
attachments = [self._url + '{0}'.format(i) for i in attachments]
else:
attributes[line[0]] = data
return message, attributes, attachments
def delete(self, msg_id):
"""
Deletes message thread (!!!message + all replies!!!) from logbook.
It also deletes all of attachments of corresponding messages from the server.
:param msg_id: message to be deleted
:return:
"""
request_headers = dict()
if self._user or self._password:
request_headers['Cookie'] = self._make_user_and_pswd_cookie()
try:
self._check_if_message_on_server(msg_id) # check if something to delete
response = requests.get(self._url + str(msg_id) + '?cmd=Delete&confirm=Yes', headers=request_headers,
allow_redirects=False, verify=False)
_validate_response(response) # raises exception if any other error identified
except requests.RequestException as e:
# If here: message is on server but cannot be downloaded (should never happen)
raise LogbookServerProblem('Cannot access logbook server to delete the message with ID: ' + str(msg_id) +
'because of:\n' + '{0}'.format(e))
# Additional validation: If successfully deleted then status_code = 302. In case command was not executed at
# all (not English language --> no download command supported) status_code = 200 and the content is just a
# html page of this whole message.
if response.status_code == 200:
raise LogbookServerProblem('Cannot process delete command (only logbooks in English supported).')
def search(self, search_term, n_results = 20, scope="subtext"):
"""
Searches the logbook and returns the message ids.
"""
request_headers = dict()
if self._user or self._password:
request_headers['Cookie'] = self._make_user_and_pswd_cookie()
# Putting n_results = 0 crashes the elog. also in the web-gui.
n_results = 1 if n_results < 1 else n_results
params = {
"mode": "full",
"reverse": "1",
"npp": n_results,
scope: search_term
}
try:
response = requests.get(self._url, params=params, headers=request_headers,
allow_redirects=False, verify=False)
# Validate response. If problems Exception will be thrown.
_validate_response(response)
resp_message = response
except requests.RequestException as e:
# If here: message is on server but cannot be downloaded (should never happen)
raise LogbookServerProblem('Cannot access logbook server to read message ids '
'because of:\n' + '{0}'.format(e))
from lxml import html
tree = html.fromstring(resp_message.content)
message_ids = tree.xpath('(//tr/td[@class="list1" or @class="list2"][1])/a/@href')
message_ids = [int(m.split("/")[-1]) for m in message_ids]
return message_ids
def get_last_message_id(self):
ids = self.get_message_ids()
if len(ids) > 0:
return ids[0]
else:
return None
def get_message_ids(self):
request_headers = dict()
if self._user or self._password:
request_headers['Cookie'] = self._make_user_and_pswd_cookie()
try:
response = requests.get(self._url + 'page', headers=request_headers,
allow_redirects=False, verify=False)
# Validate response. If problems Exception will be thrown.
_validate_response(response)
resp_message = response
except requests.RequestException as e:
# If here: message is on server but cannot be downloaded (should never happen)
raise LogbookServerProblem('Cannot access logbook server to read message ids '
'because of:\n' + '{0}'.format(e))
from lxml import html
tree = html.fromstring(resp_message.content)
message_ids = tree.xpath('(//tr/td[@class="list1" or @class="list2"][1])/a/@href')
message_ids = [int(m.split("/")[-1]) for m in message_ids]
return message_ids
def _check_if_message_on_server(self, msg_id):
"""Try to load page for specific message. If there is a htm tag like <td class="errormsg"> then there is no
such message.
:param msg_id: ID of message to be checked
:return:
"""
request_headers = dict()
if self._user or self._password:
request_headers['Cookie'] = self._make_user_and_pswd_cookie()
try:
response = requests.get(self._url + str(msg_id), headers=request_headers, allow_redirects=False,
verify=False)
# If there is no message code 200 will be returned (OK) and _validate_response will not recognise it
# but there will be some error in the html code.
resp_message, resp_headers, resp_msg_id = _validate_response(response)
# If there is no message, code 200 will be returned (OK) but there will be some error indication in
# the html code.
if re.findall('<td.*?class="errormsg".*?>.*?</td>',
resp_message.decode('utf-8', 'ignore'),
flags=re.DOTALL):
raise LogbookInvalidMessageID('Message with ID: ' + str(msg_id) + ' does not exist on logbook.')
except requests.RequestException as e:
raise LogbookServerProblem('No response from the logbook server.\nDetails: ' + '{0}'.format(e))
def _add_base_msg_attributes(self, data):
"""
Adds base message attributes which are used by all messages.
:param data: dict of current attributes
:return: content string
"""
data['cmd'] = 'Submit'
data['exp'] = self.logbook
if self._user:
data['unm'] = self._user
if self._password:
data['upwd'] = self._password
def _prepare_attachments(self, files):
"""
Parses attachments to content objects. Attachments can be:
- file like objects: must have method read() which returns bytes. If it has attribute .name it will be used
for attachment name, otherwise generic attribute<i> name will be used.
- path to the file on disk
Note that if attachment is is an url pointing to the existing Logbook server it will be ignored and no
exceptions will be raised. This can happen if attachments returned with read_method are resend.
:param files: list of file like objects or paths
:return: content string
"""
prepared = list()
i = 0
objects_to_close = list() # objects that are created (opened) by elog must be later closed
for file_obj in files:
if hasattr(file_obj, 'read'):
i += 1
attribute_name = 'attfile' + str(i)
filename = attribute_name # If file like object has no name specified use this one
candidate_filename = os.path.basename(file_obj.name)
if filename: # use only if not empty string
filename = candidate_filename
elif isinstance(file_obj, str):
# Check if it is:
# - a path to the file --> open file and append
# - an url pointing to the existing Logbook server --> ignore
filename = ""
attribute_name = ""
if os.path.isfile(file_obj):
i += 1
attribute_name = 'attfile' + str(i)
file_obj = builtins.open(file_obj, 'rb')
filename = os.path.basename(file_obj.name)
objects_to_close.append(file_obj)
elif not file_obj.startswith(self._url):
raise LogbookInvalidAttachmentType('Invalid type of attachment: \"' + file_obj + '\".')
else:
raise LogbookInvalidAttachmentType('Invalid type of attachment[' + str(i) + '].')
prepared.append((attribute_name, (filename, file_obj)))
return prepared, objects_to_close
def _make_user_and_pswd_cookie(self):
"""
prepares user name and password cookie. It is sent in header when posting a message.
:return: user name and password value for the Cookie header
"""
cookie = ''
if self._user:
cookie += 'unm=' + self._user + ';'
if self._password:
cookie += 'upwd=' + self._password + ';'
return cookie
def _remove_reserved_attributes(attributes):
"""
Removes elog reserved attributes (from the attributes dict) that can not be sent.
:param attributes: dictionary of attributes to be cleaned.
:return:
"""
if attributes:
attributes.get('$@MID@$', None)
attributes.pop('Date', None)
attributes.pop('Attachment', None)
attributes.pop('Text', None) # Remove this one because it will be send attachment like
def _replace_special_characters_in_attribute_keys(attributes):
"""
Replaces special characters in elog attribute keys by underscore, otherwise attribute values will be erased in
the http request. This is using the same replacement elog itself is using to handle these cases
:param attributes: dictionary of attributes to be cleaned.
:return: attributes with replaced keys
"""
return {re.sub('[^0-9a-zA-Z]', '_', key): value for key, value in attributes.items()}
def _validate_response(response):
""" Validate response of the request."""
msg_id = None
if response.status_code not in [200, 302]:
# 200 --> OK; 302 --> Found
# Html page is returned with error description (handling errors same way as on original client. Looks
# like there is no other way.
err = re.findall('<td.*?class="errormsg".*?>.*?</td>',
response.content.decode('utf-8', 'ignore'),
flags=re.DOTALL)
if len(err) > 0:
# Remove html tags
# If part of the message has: Please go back... remove this part since it is an instruction for
# the user when using browser.
err = re.sub('(?:<.*?>)', '', err[0])
if err:
raise LogbookMessageRejected('Rejected because of: ' + err)
else:
raise LogbookMessageRejected('Rejected because of unknown error.')
# Other unknown errors
raise LogbookMessageRejected('Rejected because of unknown error.')
else:
location = response.headers.get('Location')
if location is not None:
if 'has moved' in location:
raise LogbookServerProblem('Logbook server has moved to another location.')
elif 'fail' in location:
raise LogbookAuthenticationError('Invalid username or password.')
else:
# returned locations is something like: '<host>/<sub_dir>/<logbook>/<msg_id><query>
# with urllib.parse.urlparse returns attribute path=<sub_dir>/<logbook>/<msg_id>
msg_id = int(urllib.parse.urlsplit(location).path.split('/')[-1])
if b'form name=form1' in response.content or b'type=password' in response.content:
# Not to smart to check this way, but no other indication of this kind of error.
# C client does it the same way
raise LogbookAuthenticationError('Invalid username or password.')
return response.content, response.headers, msg_id
def _handle_pswd(password, encrypt=True):
"""
Takes password string and returns password as needed by elog. If encrypt=True then password will be
sha256 encrypted (salt='', rounds=5000). Before returning password, any trailing $5$$ will be removed
independent off encrypt flag.
:param password: password string
:param encrypt: encrypt password?
:return: elog prepared password
"""
if encrypt and password:
from passlib.hash import sha256_crypt
return sha256_crypt.encrypt(password, salt='', rounds=5000)[4:]
elif password and password.startswith('$5$$'):
return password[4:]
else:
return password